{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-02-15T13:56:11.622092Z",
     "start_time": "2019-02-15T13:56:11.614901Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "os.environ['KMP_DUPLICATE_LIB_OK']='True'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Overview"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1、数据预处理\n",
    "\n",
    "2、CNN建模和评估\n",
    "\n",
    "3、预测结果+提交"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 导入库"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-02-15T13:56:19.002329Z",
     "start_time": "2019-02-15T13:56:14.061190Z"
    },
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.image as mpimg\n",
    "import seaborn as sns\n",
    "%matplotlib inline\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import confusion_matrix\n",
    "import itertools\n",
    "\n",
    "from keras.utils.np_utils import to_categorical\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPool2D\n",
    "from keras.optimizers import RMSprop\n",
    "from keras.preprocessing.image import ImageDataGenerator\n",
    "from keras.callbacks import ReduceLROnPlateau\n",
    "\n",
    "sns.set(style='white', context='notebook', palette='deep')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 数据预处理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-02-15T13:56:24.431543Z",
     "start_time": "2019-02-15T13:56:19.049600Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train = pd.read_csv('/Users/admin/Desktop/Kaggle_data/mnist/train.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-02-15T13:56:27.944627Z",
     "start_time": "2019-02-15T13:56:24.434879Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test = pd.read_csv('/Users/admin/Desktop/Kaggle_data/mnist/test.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## sns统计每种标签数量"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-02-15T13:56:29.482041Z",
     "start_time": "2019-02-15T13:56:29.026458Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZMAAAEPCAYAAACHuClZAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAE+5JREFUeJzt3XuwXWV5x/Fvcs6JpmBAJmCCAYRBHtQWgnJpyy0qyKAoYsUIEQgKyCAOU1FQCeUyxbaMBsUSZeQSbBBoQawjibeoEIrghdso8pRWRSPJoKJyM5CTpH+sdciGJrBz3rPWPif5fmYynP3stfM+JCfnt9/17vWucWvWrEGSpBLje92AJGnsM0wkScUME0lSMcNEklTMMJEkFTNMJEnFDBNJUjHDRJJUzDCRJBUzTCRJxQwTSVKx/l430JSIeBGwF7AMWNXjdiRprOgDpgI/zMynun3RRhsmVEGypNdNSNIYtT9wa7cHb8xhsgzg6quvZsqUKb3uRZLGhOXLlzNr1iyof4Z2a2MOk1UAU6ZMYdq0ab3uRZLGmg1aHnABXpJUzDCRJBUzTCRJxQwTSVIxw0SSVMwwkSQVM0wkScUMk5atHly5UY4ladO2MV+0OCqN7x/gxxee0MpYrzvjslbGkSRnJpKkYoaJJKmYYSJJKmaYSJKKGSaSpGKGiSSpmGEiSSpmmEiSihkmkqRihokkqZhhIkkqZphIGlUGBwc3yrE2dm70KGlU6e/v51Of+lQrY51++umtjLMpcGainln1dHtb5Lc5lrQpcmainumbMMDCY49vZaw3f/HKVsaRNlXOTCRJxQwTSVIxw0SSVMwwkSQVM0wkScUME0kapVauWj1mxvKjwZuopwdXMqF/YKMZR9oYDfSN50M33tzKWHOPOLDo9YbJJmpC/wCzrzyt8XHmH/+ZxsfQyFk9uIrx/X0bzThqzyYVJk+vXMWEgea/gdsaRxpp4/v7uGfe9xofZ/dTZjQ+htq1SYXJhIE+jj7j6sbH+dKFsxofQ5JGk8bDJCI+CUzOzNkRMR24DJgE3AKcnJmDEbE9sADYBkhgVmY+HhFbAlcDOwG/Bd6Vmcub7lmblsGVq+hvYSbZ1jhSLzQaJhHxRuA44Ka6tAA4ITNvj4jLgROBzwHzgHmZeW1EnA2cDZwJ/COwJDPfEhHHAJ8BZjbZszY9/QN9fOKs6xsf5+MXvLPxMTRyVq9ayfi+5j880tY4TWssTCJiK+AC4BPA7hGxAzAxM2+vD5kPnBcRlwEHAG/vqN9MFSZvqZ8DuAa4JCIGMtMtYCU1anzfALd87dzGxzngsObHaEOT15lcCpwF/KF+vC2wrOP5ZcA0YDLwaGYOPqf+rNfUzz8KbN1gz5KkYWgkTCLiBODXmbn4OWOt6Xg8Dli9jjp1feiYTuM6npMkjRJNneaaCUyNiLuBrYDNqQJjascxU4CHgIeBLSKiLzNX1cc8VB/zm/q4pRHRD7wE+H1DPUuShqmRmUlmHpyZf5mZ04F/AL6amccDKyJi3/qwY4BF9frHEtYurB8LLKq/Xlg/pn5+ieslkjT6tH2dySzgCxExCbgTuLiunwJcFRFzgF8BR9X1s4H5EfFT4I/16yVJo0zjYZKZ86k+oUVm3gPsvY5jHgRmrKP+CPC2RhuUJBVz12BJUjHDRJJUzDCRJBUzTCRJxQwTaRQYXNneJ97bHEubjk1qC3pptOofGGDux97fylgf+qdLWxlHmxZnJpKkYoaJJKmYYSJJKmaYSJKKGSaSpGKGiSSpmGEiSSpmmEiSihkmkqRihokkqZhhIkkqZphIkooZJpKkYoaJJKmYYSJJKmaYSJKKGSaSpGKGiSSpmGEiSSpmmEiSihkmkqRihokkqZhhIkkqZphIkooZJpKkYoaJJKmYYSJJKmaYSJKKGSaSpGL9Tf7mEXE+8E5gDXB5Zs6NiIOAucBE4LrMnFMfOx24DJgE3AKcnJmDEbE9sADYBkhgVmY+3mTfkqQN09jMJCIOBN4A7AbsCXwwInYHrgAOB14F7BURh9YvWQCcmpm7AOOAE+v6PGBeZu4K/Ag4u6meJUnD01iYZObNwOszc5BqVtEPbAk8kJm/qOsLgCMjYgdgYmbeXr98fl0fAA4Aru+sN9WzJGl4Gl0zycyVEXEecB+wGNgWWNZxyDJg2vPUJwOP1sHTWZckjSKNL8Bn5jnA1sB2wC5U6ydDxgGr6z66qVPXJUmjSJNrJrvWi+pk5pPAl4EZwNSOw6YADwFL11N/GNgiIvrq+tS6LkkaRZqcmewEfCEiXhQRE6gW3S8FIiJ2rgPiaGBRZj4IrIiIfevXHlPXVwJLgJl1/VhgUYM9S5KGockF+IXATcBdwI+B2zLzWmA2cAPVOsr9rF1cnwVcFBH3A5sDF9f1U4CTIuI+YH9gTlM9S5KGp9HrTDLzXODc59QWA7uv49h7gL3XUX+Q6vSYJGmU8gp4SVIxw0SSVMwwkSQVM0wkScUME0lSMcNEklSsqzCJiJevo/bqkW9HkjQWPe91JhGxVf3lwoiYQbVnFsAA1fYouzbXmiRprHihixavAQ6uv/59R32QtVeuS5I2cc8bJpl5CEBEXJGZ722nJUnSWNPVdiqZ+d76BlZbsfZUF5l5Z1ONSZLGjq7CpL7B1UeotoQfur/IGqqdgSVJm7huN3o8Ftg5M72XiCTp/+n2OpNfGySSpPXpdmayOCIuBP4T+PNQ0TUTSRJ0Hyaz6/8e2VFzzUSSBHT/aa4dm25EkjR2dftprg+tq56Zc0e2HUnSWNTtaa6/6vh6AnAgsHjk25EkjUXdnuY6vvNxRGwLXN5IR5KkMWdYW9DXHxN+xci2Ikkaq4azZjIO2JPqanhJkoa1ZrIG+BXV9iqSJG3Ymkm92eNAZv5Po11JksaUbk9z7Ux19fu2wPiI+B1wWGb+rMnmJEljQ7cL8P8KXJiZL83MLYB/BC5pri1J0ljSbZi8LDOvGnqQmVcCWzfTkiRprOk2TPo77gdPRExm7X1NJEmbuG4/zfVZ4PaIuI4qRN4NXNRYV5KkMaXbmclCqhCZALwaeDlwY1NNSZLGlm7DZD5wSWaeCbwHOAu4oqmmJEljS7dhMjkzLwbIzBWZ+WlganNtSZLGkg1ZgN926EFEvIxqWxVJkrpegJ8L3B0RX6daOzkIt1ORJNW6mplk5hVUAXIX8CPgkMz8UpONSZLGjm5nJmTmvcC9G/KbR8Q5wLvqhzdl5hkRcRDVTGcicF1mzqmPnQ5cBkwCbgFOzszBiNgeWABsAyQwKzMf35A+JEnNGtb9TLpRh8abgD2A6cDrIuIoqk+BHQ68CtgrIg6tX7IAODUzd6Fajzmxrs8D5mXmrlSzorOb6lmSNDyNhQmwDDg9M5/OzJXAz4BdgAcy8xeZOUgVIEfWuxFPzMzb69fOr+sDwAHA9Z31BnuWJA1D16e5NlRm/nTo64h4JdXprs9ShcyQZcA0qt2I11WfDDxaB09nXZI0ijQ5MwEgIl4DfIvq018/59l7eo0DVtd9dFOnrkuSRpFGwyQi9gUWAx+tdx1eyrMvdpwCPPQ89YeBLSKir65PreuSpFGkyQX47YCvAEdn5rV1+Y7qqdi5DoijgUWZ+SCwog4fgGPq+kpgCTCzrh8LLGqqZ0nS8DS2ZgJ8GHgxMDcihmqfB2YDN9TPLWTt4vos4AsRMQm4E7i4rp8CXBURc6juPX9Ugz1LkoahyQX404DT1vP07us4/h5g73XUHwRmjGhzkqQR1fgCvCRp42eYSJKKGSaSpGKGiSSpmGEiSSpmmEiSihkmkqRihokkqZhhIkkqZphIkooZJpKkYoaJJKmYYSJJKmaYSJKKGSaSpGKGiSSpmGEiSSpmmEiSihkmkqRihokkqZhhIkkqZphIkooZJpKkYoaJJKmYYSJJKmaYSJKKGSaSpGKGiSSpmGEiSSpmmEiSihkmkqRihokkqZhhIkkqZphIkooZJpKkYv1NDxARk4DbgMMy85cRcRAwF5gIXJeZc+rjpgOXAZOAW4CTM3MwIrYHFgDbAAnMyszHm+5bktS9RmcmEbEPcCuwS/14InAFcDjwKmCviDi0PnwBcGpm7gKMA06s6/OAeZm5K/Aj4Owme5YkbbimT3OdCHwAeKh+vDfwQGb+IjMHqQLkyIjYAZiYmbfXx82v6wPAAcD1nfWGe5YkbaBGT3Nl5gkAETFU2hZY1nHIMmDa89QnA4/WwdNZlySNIm0vwI8H1nQ8Hges3oA6dV2SNIq0HSZLgakdj6dQnQJbX/1hYIuI6KvrU1l7ykySNEq0HSZ3ABERO9cBcTSwKDMfBFZExL71ccfU9ZXAEmBmXT8WWNRyz5KkF9BqmGTmCmA2cANwH3A/axfXZwEXRcT9wObAxXX9FOCkiLgP2B+Y02bPkqQX1vh1JgCZ+YqOrxcDu6/jmHuoPu313PqDwIwG25MkFfIKeElSMcNEklTMMJEkFTNMJEnFDBNJUjHDRJJUzDCRJBUzTCRJxQwTSVIxw0SSVMwwkSQVM0wkScUME0lSMcNEklTMMJEkFTNMJEnFDBNJUjHDRJJUzDCRJBUzTCRJxQwTSVIxw0SSVMwwkSQVM0wkScUME0lSMcNEklTMMJEkFTNMJEnFDBNJUjHDRJJUzDCRJBUzTCRJxQwTSVIxw0SSVMwwkSQVM0wkScX6e91ANyLiaGAOMAB8OjMv6XFLkqQOo35mEhEvBy4A9gOmAydFxKt725UkqdNYmJkcBHwnMx8BiIjrgXcC57/A6/oAli9f/qziU0/+sYEWn23p0qXP+/xvH1vReA/d9LHij0/2vIdHnhodfxaPP/GHnvfw2BN/bryHbvp4+NHf9byHxx57rPEeuunjd4883vMennyk+b+Pzj46fmb2bcjrx61Zs2aEWxpZEfExYLPMnFM/PgHYOzNPeoHX7QcsaaFFSdoY7Z+Zt3Z78FiYmYwHOhNvHLC6i9f9ENgfWAasaqAvSdoY9QFTqX6Gdm0shMlSqlAYMgV46IVelJlPAV2nqiTpGf+7oS8YC2HybeDciNgaeAL4O+B5T3FJkto16j/NlZm/Ac4CvgvcDXwpM3/Q264kSZ1G/QK8JGn0G/UzE0nS6GeYSJKKGSaSpGKGiSSp2Fj4aHDPjJYNJiNiEnAbcFhm/rIH458DvKt+eFNmntF2D3Uf51NtpbMGuDwz5/aij7qXTwKTM3N2j8b/LrANsLIuvT8z72i5h7cC5wCbAd/MzNPaHL/u4QTg1I7SjsC/Zeap63lJU328B/hY/XBRZn64zfE7+vgocDzwFHBdZl7Q1tjOTNZjtGwwGRH7UF18uUvbY9fjHwS8CdiD6s/hdRFxRA/6OBB4A7AbsCfwwYiItvuoe3kjcFwvxq7HH0f1/bB7Zk6vf7UdJDsBnwfeTvV38tqIOLTNHgAy87KhPwNgFvAwcG6bPUTEXwAXAwcCuwP71/9uWlWPeTSwF9W/130i4h1tjW+YrN8zG0xm5hPA0AaTbTsR+ABdXPXfkGXA6Zn5dGauBH4GbN92E5l5M/D6zBykekfeT3URa6siYiuqNxmfaHvszjbq/34zIu6JiFbfhdeOoHrnu7T+vpgJtBpo6/A54OOZ2c7OiGv1Uf0s3YzqLMYA0M6unc+2B/CNzHw0M1cBX6cK+1YYJuu3LdUP0iHLgGltN5GZJ2RmzzaszMyfZubtABHxSqrTXQt71MvKiDgPuA9YDPymB21cSnURbfPbDK/fS6n+/48A3gicHBEHt9zDzkBfRHw1Iu4GTqGHfyb1u/KJmfkfbY+dmY8BZwP3U23/9Euq09JtuxM4JCK2iogXA2+j2n6qFYbJ+g13g8mNUkS8BvgW8JHMfKBXfWTmOcDWwHZUs7bW1Ofnf52Zi9sc97ky8/uZeWxm/ql+F3458OaW2+inmr2/D/gbYB96eOoPeD/QkzW0iNgNeC+wA9Wb0FVA62sm9fflfOB7VLOSW4Gn2xrfMFm/pVQ7Zw7paoPJjVFE7Ev1TvijmXlVj3rYNSKmA2Tmk8CXqc7Vt2km8Kb6nfj5wNsi4qKWeyAi9qvXbYaMY+1CfFuWA9/OzN9m5p+BG4G9W+4BgIiYQLVe8dVejA8cAizOzIfrDWbnAzPabiIiXgLckJm7ZeYMqkX4Dd6wcbj8NNf6ucEkEBHbAV8BZmbmd3rYyk7AefV9atYAhwNXtNlAZj5zKikiZgMzMvPv2+yhtiVwfkT8LdX5+eOAk1vu4WvAVRGxJfAYcCjV90kv7Ab8d7222Qv3ABdGxGbAk8Bb2cDt20fIjsAXI2JPqvWb99W/WuHMZD3cYPIZHwZeDMyNiLvrX23/4CIzFwI3AXcBPwZuy8xr2+5jNMjMr/HsP4srMvP7LfdwB3Ah1amU+4AHgSvb7KHDTlRnEnoiM78JXEP1d3EvVcD/cw/6uBe4oe7hB1SXM/xXW+O70aMkqZgzE0lSMcNEklTMMJEkFTNMJEnFDBNJUjHDRBohETEjIn7yAsesiYjJG/j7zo+InuxCK3XLMJEkFfMKeGmERcQuwCXAS6i25LmbageBFfUhF0TEXlRv5ubUFyESEe+j2jBxPPB74NTMvL/t/qXhcGYijbwTgasy86+pdtfdEXhLx/M/z8zXAu+h2pJk6/p+LccB+2fmHlRXl9/Yct/SsDkzkUbemcDBEXEG1U2stgU273j+8wCZ+ZOIuI9q1939qILnto57fr20vn+KNOoZJtLIu4bq39a/U+2htT3Vzr5DVnV8PZ5qx98+qtvNngkQEeOpQqiX902RuuZpLmnkHQKcn5nX1Y/3oQqLIbMBIuK1VLORO4BvAEdFxNBtD06m2vZfGhOcmUgj7+PAjRHxBPAn4Gaq0BiyU0TcRbWV/rsz8xGqW/D+C/CtiFgNPAq8IzPX9OhW99IGcddgSVIxT3NJkooZJpKkYoaJJKmYYSJJKmaYSJKKGSaSpGKGiSSpmGEiSSr2fy5dZTfrIvsDAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "Y_train = train['label']\n",
    "\n",
    "X_train = train.drop(['label'], axis=1)\n",
    "\n",
    "del train\n",
    "\n",
    "g = sns.countplot(Y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4C—完整性"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-02-15T13:56:31.357681Z",
     "start_time": "2019-02-15T13:56:30.414158Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count       784\n",
       "unique        1\n",
       "top       False\n",
       "freq        784\n",
       "dtype: object"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.isnull().any().describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-02-15T13:56:32.477967Z",
     "start_time": "2019-02-15T13:56:31.996480Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count       784\n",
       "unique        1\n",
       "top       False\n",
       "freq        784\n",
       "dtype: object"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.isnull().any().describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 标准化"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-02-15T13:56:34.349809Z",
     "start_time": "2019-02-15T13:56:33.169614Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_train = X_train / 255.0\n",
    "test = test/255.0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 『image』Reshape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "【图像特征reshape】4维数组：数量、横轴、纵轴、channels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-02-15T13:56:35.356155Z",
     "start_time": "2019-02-15T13:56:35.346133Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_train = X_train.values.reshape(-1, 28, 28, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-02-15T13:56:35.847822Z",
     "start_time": "2019-02-15T13:56:35.836232Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test = test.values.reshape(-1, 28, 28, 1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4C—转换性—label encoding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-02-15T13:56:36.042158Z",
     "start_time": "2019-02-15T13:56:35.980994Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "Y_train = to_categorical(Y_train, num_classes=10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 数据集划分"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-02-15T13:56:37.839006Z",
     "start_time": "2019-02-15T13:56:36.219203Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "random_seed = 2\n",
    "\n",
    "X_train, X_val, Y_train, Y_val = train_test_split(X_train, Y_train, \n",
    "                                                 test_size=0.1, random_state=random_seed)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 「image」输出图片"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-02-15T13:56:38.129324Z",
     "start_time": "2019-02-15T13:56:37.956665Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQMAAAEBCAYAAAB8GcDAAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAADjhJREFUeJzt3X2oHfWdx/H3NYpPVdpAS2IS7YLNd7MqtcTURY19MAoWiyupZpuClfWRpougRmSNtXGxshvMBlxdxW1sikYFg0K1llqTtj6hIZi6Mdlv7BrBJHfLQqBoumqid/84v7uetrlzTu55mHOT9wuk98z3zJmv03s//mbmd2aGRkZGkKRD6m5A0mAwDCQBhoGkwjCQBBgGkgrDQBJgGEgqDANJgGEgqTAMJAGGgaTi0H5vMCIOB+YAw8CH/d6+dBCYBEwF1mfm++2u1FEYRMRCYAlwGLAiM+9uY7U5wHOdbFdSW+YCz7f75nGHQURMA24HZgPvAy9GxLrM3Nxi1WGA7Tt2s/dDvzEpdduhk4aYPu1oKH9rba/XwTbnAWszcxdARDwGfAO4rcV6HwLs/XCEvXsNA6mH9uswvJMTiMfxx8kzDEzv4PMk1aiTMDgEaP5P+xDwUWftSKpLJ2GwncYZy1FTgJ2dtSOpLp2cM/gF8P2I+DSwG5gPXNWVriT13bhHBpm5A7gZWAdsBFZn5ivdakxSf3U0zyAzVwOru9SLpBo5HVkSYBhIKgwDSYBhIKkwDCQBhoGkwjCQBBgGkgrDQBJgGEgqDANJgGEgqTAMJAGGgaTCMJAEGAaSCsNAEmAYSCoMA0mAYSCpMAwkAYaBpMIwkAQYBpIKw0ASYBhIKgwDSYBhIKkwDCQBHT6FWQeP+VPnVNZ/cMz/VtZP+OW/jVnb88idlevu2bC1sn7l2qMr62uG11fW1dBRGETEOuAzwJ6y6OrMfLnjriT13bjDICKGgJnACZm5t3stSapDJ+cMovzvzyPiNxHx3W40JKkenYTBp4BngYuAc4BrIuLcrnQlqe/GfZiQmS8BL42+jogfAl8DnulCX5L6bNwjg4g4KyLOaVo0xMcnEiVNMJ1cTfgkcFtEnAEcBnwbuKYrXUnqu04OE56MiNOBV4FJwN3l0EET0E8mn11Zn3nMrsr6P7xzZGV9zXFz97unUf849SuV9VU/Pqey/oNFY8+BiK2bxtXTgaijeQaZeQtwS5d6kVQjpyNLAgwDSYVhIAkwDCQVhoEkwK8wH1BmTZ4xZu2F84+pXPeFp6s/u85LcLcMr6usr17w28p61b/7rjl/Vbnu5Ic2V9YPJI4MJAGGgaTCMJAEGAaSCsNAEmAYSCoMA0mA8wwOKK8smT1mbee91dfiv77r191up2+27Hq7sn7m02PPv1j/6NWV6856+r6Otj2RODKQBBgGkgrDQBJgGEgqDANJgGEgqTAMJAHOM5hQWj0WfeiU08asxdYnut3OhLHw8BPHrB160pcq192ya0m32xlYjgwkAYaBpMIwkAQYBpIKw0ASYBhIKgwDSYDzDCaUhzYsr6x/a/Z1fepksLSaf7F4w23j/uz3dj5XWf/D4ur7IUyk5y60FQYRcSzwInBBZr4VEfOA5cCRwKOZefDMzJAOUC0PEyLidOB5YGZ5fSSwErgQmAXMiYjze9mkpN5r55zBlcAiYGd5/UXgjczclpl7gQeBi3vUn6Q+aXmYkJlXAETE6KLjgOGmtwwD07vemaS+Gs/VhEOAkabXQ8BH3WlHUl3GEwbbgalNr6fw8SGEpAlqPJcWXwYiIk4EtgELaZxQlDSB7XcYZOZ7EXEZsAY4Avgp8FiX+zootbpe3sqa4fVd6mSwtNovreZfdKLVPIIzn36nZ9vut7bDIDM/2/Tzs8Dne9GQpHo4HVkSYBhIKgwDSYBhIKkwDCQBfoV5oJzKJ+puoWeqLg/+aPG0ynUP+9vrO9r2nkfuHLN22bIdleuuGZ44X0HulCMDSYBhIKkwDCQBhoGkwjCQBBgGkgrDQBLgPIOBspF3O1q/6lp+p19vnjV5RmX9lSWzK+tVcwX2vv6rynWfOfnmyvqNbKusb9n1dmVdDY4MJAGGgaTCMJAEGAaSCsNAEmAYSCoMA0mA8wwGSqu5APe3uG33QxvuG7O2+eSFlev+M39RWT930+2V9VaWzf7emLVbhtd19NnqDkcGkgDDQFJhGEgCDANJhWEgCTAMJBWGgSTAeQYTyuSHqu/h/96ysWuvblrd0ba9p8CBr+0wiIhjgReBCzLzrYh4ADgL2F3esjQzH+9Bj5L6oK0wiIjTgfuBmU2LTwPOzszhXjQmqb/aPWdwJbAI2AkQEUcBxwMrI+K1iFgaEZ5/kCawtv6AM/OKzHyuadEUYC3wd8BfA3OBy7vfnqR+GdcJxMx8E7ho9HVE3AVcSuNQQtIENK6hfUScEhHzmxYNAXu605KkOoz30uIQsCIi1gLvAlcBq7rWlaS+G+9hwmsRcQfwAnAYsCYzH+5qZ/ozP5l89rjX3fPInZX1qucagPMIDgb7FQaZ+dmmn+8B7ul2Q5Lq4eVASYBhIKkwDCQBhoGkwjCQBPgV5oHyzvK/qazv2bC1sv6t2deNWWt1G/b5y3ZU1tc/Wn2b9h2LHqmsx9ZNlXXVz5GBJMAwkFQYBpIAw0BSYRhIAgwDSYVhIAlwnkFXzZo8o7K+8PATK+ut5hG0ulV6J1rNQ9i84L8r6y1vxX7c3P1tSX3myEASYBhIKgwDSYBhIKkwDCQBhoGkwjCQBDjPoKtafef/Xy59trLey3kEnfJW6Ac+RwaSAMNAUmEYSAIMA0mFYSAJMAwkFYaBJKDNeQYRcStwSXn5VGbeGBHzgOXAkcCjmbmkRz0OlE4ei37L8LoudtJfre7VoImv5cig/NGfB3wBOBWYHRHfBFYCFwKzgDkRcX4vG5XUW+0cJgwD12fmB5m5B9gCzATeyMxtmbkXeBC4uId9SuqxlocJmfn66M8R8Tkahwt30QiJUcPA9K53J6lv2j6BGBEnAc8Ai4E3gZGm8hDwUXdbk9RPbYVBRJwJPAvclJmrgO3A1Ka3TAF2dr89Sf3S8jAhImYATwALMnNtWfxyoxQnAtuAhTROKEqaoNq5tHgDcASwPCJGl90LXAasKbWfAo/1oL8JpdVXlAdZq0uHrb6evff1X3WzHdWgnROI1wLXjlH+fHfbkVQXZyBKAgwDSYVhIAkwDCQVhoEkwDCQVHir9P107qbbx6z9aPZ1fexk/8yfOqeyvurHF3b0+XMW3NfR+qqfIwNJgGEgqTAMJAGGgaTCMJAEGAaSCsNAEuA8g/22bPb3xqy1ulZ/6qWf6Gjbi776u8r6UcvGf63/mZNvrqx/fdevx/3ZmhgcGUgCDANJhWEgCTAMJBWGgSTAMJBUGAaSAOcZ7Leqx6pfsuh/Ktdd9NXqJ9C1mifwh8XVzy6omgOx+v3fVq67ZdfblXUd+BwZSAIMA0mFYSAJMAwkFYaBJMAwkFQYBpKANucZRMStwCXl5VOZeWNEPACcBewuy5dm5uM96HHCiK2bqt+wtcUHPDS3ww42d7i+DmYtwyAi5gHnAV8ARoCfRcRFwGnA2Zk53NsWJfVDOyODYeD6zPwAICK2AMeXf1ZGxDTgcRojg+opdpIGVsswyMzXR3+OiM/ROFyYC3wZ+A7we+BJ4HLg/p50Kann2v5uQkScBDwFLM7MBC5qqt0FXIphIE1YbV1NiIgzgWeBmzJzVUScEhHzm94yBOzpRYOS+qOdE4gzgCeABZm5tiweAlZExFrgXeAqYFXPupTUc+0cJtwAHAEsj4jRZfcCdwAvAIcBazLz4Z50KKkv2jmBeC1w7Rjle7rbjqS6OANREmAYSCoMA0mAYSCpMAwkAYaBpMIwkAQYBpIKw0ASYBhIKgwDSYBhIKkwDCQB9TyFeRLAoZOGati0dOBr+tuatF/rdb+VlqYCTJ92dA2blg4qU4H/avfNdYTBeho3VB0GPqxh+9KBbhKNIFi/PysNjYyM9KYdSROKJxAlAYaBpMIwkAQYBpIKw0ASYBhIKgwDSUA9k47+X0QsBJbQeCrTisy8u85+mkXEOuAzfPwMyasz8+UaWyIijgVeBC7IzLciYh6wHDgSeDQzlwxIXw8AZwG7y1uWZubjNfR1K42nhgM8lZk3DtA+21dvte632iYdRcQ04HlgNvA+jV+mb2bm5loaahIRQ8B24ITM3Ft3PwARcTqNp1z/JTAT+B2QwJeAt2k8IXtFZj5dZ18lDP4DOC8zh/vZy5/0NQ9YCnwFGAF+Bvw78E/Uv8/21du/ArdR436r8zBhHrA2M3dl5m7gMeAbNfbTbPShkj+PiN9ExHdr7abhSmARsLO8/iLwRmZuK4H1IHBx3X1FxFHA8cDKiHgtIpZGRB2/Z8PA9Zn5QWbuAbbQCNFB2Gf76u14at5vdR4mHEdjp4wapvELPgg+ReMR9H9P4xDmlxGRmflMXQ1l5hUATQ+/3df+m97ntvbV1xRgLfAd4PfAk8DlNEYP/ezr9dGfI+JzNIbkdzEY+2xfvc0FvkyN+63OMDiExhBp1BDwUU29/JHMfAl4afR1RPwQ+BpQWxjsw0Duv8x8E7ho9HVE3AVcSp/DoGn7J9E4HFgM7KUxOhhV6z5r7i0zk5r3W52HCdspX2cupvDxELhWEXFWRJzTtGiIj08kDoqB3H8RcUpEzG9aVNu+i4gzaYzwbsrMVQzQPvvT3gZhv9U5MvgF8P2I+DSNs6fzgatq7KfZJ4HbIuIMGocJ3wauqbelP/MyEBFxIrANWAisrLcloPFLvCIi1gLv0vj/dFW/m4iIGcATwILMXFsWD8Q+G6O32vdbbSODzNwB3AysAzYCqzPzlbr6aZaZT9IYvr0KbABWlkOHgZGZ7wGXAWuAzcB/0jgJW6vMfA24A3iBRl8bM/PhGlq5ATgCWB4RGyNiI439dRn177N99XYGNe8372cgCXAGoqTCMJAEGAaSCsNAEmAYSCoMA0mAYSCpMAwkAfB//VKVnbQ0hVQAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "g = plt.imshow(X_train[0][:,:,0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "run_control": {
     "marked": false
    }
   },
   "source": [
    "# CNN model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1、Conv2D层 => 图像的转化 => 减少参数\n",
    "\n",
    "2、Maxpooling池化层 => 降维过滤器 => 作用：减少计算复杂度+减少过拟合\n",
    "\n",
    "3、Dropout层 => 正则化方法 => 随机忽视一些结点 => 作用：减少过拟合\n",
    "\n",
    "4、Relu激活函数（中间结点） + softmax（最终分类）\n",
    "\n",
    "5、Flatten layer => 转换特征图到1维的向量\n",
    "\n",
    "6、全连接层：Dense"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Arch: [Conv2D/relu -> Conv2D/relu -> Maxpool2D -> Dopout] * 2 -> Faltten -> Dense -> \n",
    "\n",
    "Dropout -> Out"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 模型架构"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-02-15T13:56:38.372028Z",
     "start_time": "2019-02-15T13:56:38.254701Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = Sequential()\n",
    "\n",
    "model.add(Conv2D(filters=32, kernel_size=(5,5), padding='Same', \n",
    "                 activation='relu', input_shape = (28, 28, 1) ))\n",
    "\n",
    "model.add(Conv2D(filters=32, kernel_size=(5,5), padding='Same', activation='relu'))\n",
    "\n",
    "model.add(MaxPool2D(pool_size=(2,2)))\n",
    "\n",
    "model.add(Dropout(0.25))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-02-15T13:56:38.569467Z",
     "start_time": "2019-02-15T13:56:38.487212Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.add(Conv2D(filters=64, kernel_size=(3,3), padding='Same', \n",
    "                 activation='relu', input_shape = (28, 28, 1) ))\n",
    "\n",
    "model.add(Conv2D(filters=64, kernel_size=(3,3), padding='Same', activation='relu'))\n",
    "\n",
    "model.add(MaxPool2D(pool_size=(2,2), strides=(2,2)))\n",
    "\n",
    "model.add(Dropout(0.25))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-02-15T13:56:38.700373Z",
     "start_time": "2019-02-15T13:56:38.677766Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.add(Flatten())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-02-15T13:56:38.864325Z",
     "start_time": "2019-02-15T13:56:38.828281Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.add(Dense(256, activation='relu'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-02-15T13:56:39.024091Z",
     "start_time": "2019-02-15T13:56:38.989927Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.add(Dropout(0.5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-02-15T13:56:39.174565Z",
     "start_time": "2019-02-15T13:56:39.140434Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.add(Dense(10, activation='softmax'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 优化器 + 损失 + 评估指标"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-02-15T13:56:39.314759Z",
     "start_time": "2019-02-15T13:56:39.282210Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "optimizer = RMSprop(lr=0.001, rho=0.9, epsilon=1e-08, decay=0.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-02-15T13:56:39.477467Z",
     "start_time": "2019-02-15T13:56:39.430354Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.compile(optimizer=optimizer, loss='categorical_crossentropy', metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 🌲『快速收敛』annealing method 退火方法"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-02-15T13:56:39.613940Z",
     "start_time": "2019-02-15T13:56:39.605835Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "learning_rate_reduction = ReduceLROnPlateau(monitor='val_acc', patience=3, verbose=1,\n",
    "                                            factor=0.5, min_lr=0.00001)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-02-15T13:56:39.763336Z",
     "start_time": "2019-02-15T13:56:39.755779Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "epochs=5\n",
    "batch_size=86"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 🌲数据集 => 数据增强 Data augmentation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "目的：\n",
    "\n",
    "\n",
    "技巧：\n",
    "\n",
    "\n",
    "结果："
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-02-15T13:56:40.123171Z",
     "start_time": "2019-02-15T13:56:39.891242Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "datagen = ImageDataGenerator(\n",
    "        featurewise_center=False,  # set input mean to 0 over the dataset\n",
    "        samplewise_center=False,  # set each sample mean to 0\n",
    "        featurewise_std_normalization=False,  # divide inputs by std of the dataset\n",
    "        samplewise_std_normalization=False,  # divide each input by its std\n",
    "        zca_whitening=False,  # apply ZCA whitening\n",
    "        rotation_range=10,  # randomly rotate images in the range (degrees, 0 to 180)\n",
    "        zoom_range = 0.1, # Randomly zoom image \n",
    "        width_shift_range=0.1,  # randomly shift images horizontally (fraction of total width)\n",
    "        height_shift_range=0.1,  # randomly shift images vertically (fraction of total height)\n",
    "        horizontal_flip=False,  # randomly flip images\n",
    "        vertical_flip=False)  # randomly flip images\n",
    "\n",
    "\n",
    "datagen.fit(X_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 训练模型，拟合"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-02-15T13:59:17.213412Z",
     "start_time": "2019-02-15T13:56:40.246787Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      " - 198s - loss: 0.4203 - acc: 0.8637 - val_loss: 0.0727 - val_acc: 0.9790\n",
      "Epoch 2/5\n",
      " - 153s - loss: 0.1295 - acc: 0.9617 - val_loss: 0.0425 - val_acc: 0.9881\n",
      "Epoch 3/5\n",
      " - 153s - loss: 0.0949 - acc: 0.9714 - val_loss: 0.0330 - val_acc: 0.9912\n",
      "Epoch 4/5\n",
      " - 154s - loss: 0.0811 - acc: 0.9761 - val_loss: 0.0362 - val_acc: 0.9898\n",
      "Epoch 5/5\n",
      " - 153s - loss: 0.0729 - acc: 0.9781 - val_loss: 0.0326 - val_acc: 0.9933\n"
     ]
    }
   ],
   "source": [
    "history = model.fit_generator(datagen.flow(X_train,Y_train, batch_size=batch_size),\n",
    "                              epochs = epochs, validation_data = (X_val,Y_val),\n",
    "                              verbose = 2, steps_per_epoch=X_train.shape[0] // batch_size\n",
    "                              , callbacks=[learning_rate_reduction])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 模型评估"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 训练和测试曲线"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.07270162285638175,\n",
       " 0.04251237820922619,\n",
       " 0.03303992559867246,\n",
       " 0.03621046427523695,\n",
       " 0.0325617210189358]"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "history.history['val_loss']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "横轴：迭代次数\n",
    "\n",
    "随着迭代次数增多，训练损失和测试损失均不断减小，且趋近相同\n",
    "\n",
    "> 验证准确度大于训练准确度：\n",
    "\n",
    "> 训练集中存在较多容易混淆的图片，导致训练集整体准确率较低"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEBCAYAAAB13qL/AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAIABJREFUeJzt3Xd8FNX6+PHPbE1CGoQSqgriUaSJQAREpVz9Kgoo6FUR5Cpio9zfvYIoSBNRwQIi4rV3rKigyNULFqSDIipyRKQYSqghjWTr74/dJJuekDIpz/v1ymtnZ87MPHuye56ZM83w+/0IIYSouyxmByCEEMJckgiEEKKOk0QghBB1nCQCIYSo4yQRCCFEHSeJQAgh6jhJBEIIUcdJIhBCiDpOEoEQQtRxkgiEEKKOk0QghBB1nM3sAAqjlHIC3YCDgNfkcIQQoqawAk2BTVrrrNLOVC0TAYEksNrsIIQQoobqDXxf2sLVNREcBHj77beJj483OxYhhKgRDh06xLBhwyDYhpZWdU0EXoD4+HhatGhhdixCCFHTlKlLvdYdLPb7/fz0+xGy3HJoQQghSqPWJYKUdBcPvbCWh55fS0q6y+xwhBCi2itX15BS6mZgCmAH5mmtFxZRbgDwrNb6rPKsrzRiIp3cP7wbT76zhYkLvmP6HT2Ij6tX2asVosbz+Xzs2rWLlJQUs0MRJXA4HDRv3pwGDRpUyPJOOxEopZoDjwAXAlnAWqXU11rr7fnKNQGeAIzyBFoWvTo1IzbKyaxXNjBhwWqm3X4RZ7eMrarVC1EjHTx4EMMwuOCCC7BYal1nQa3h8/nIyMhg165d+P1+4uLiyr3M8vy3+wOrtNbHtdbpwIfA0ELKvQTMKMd6Tsv5reOYM7Y3DpuFB577ni07kqo6BCFqlKNHj9KyZUtJAtWcxWIhMjKSNm3a8Oeff3LixInyL7Mc8zYj7ylKB4E8p/gopcYBPwDry7Ge09aySRRzx11Cs4aRzHx5A//buNeMMISoETweDw6Hw+wwRClFRERgGAYff/wxPp+vXMsqTyKwAP6Q9waQE41Sqj0wBHi4HOsotwbRYTx6by86nt2Q+e9tZfGXGr/fX/KMQtRBhlFlPbiinCwWC4ZhkJqaSmZmZvmWVY55EwlcypwtHjgQ8v764PTNwHKgmVLKlKuFI8LsTBt1EX27tuSd/+7g2Q9+wustXwYVQlSeGTNmMGjQIK666irat2/PoEGDGDRoEB999FGplzF//nxWrlxZbJlBgwaVN1Q2bNjA8OHDy72c02UYRrn3CMpz1tD/gOlKqUZAOoGt/9HZE7XW04BpAEqpM4FvtNa9y7G+crFZLfzzxgtoGBvO+//7neMpmUwc3pVwZ3W9pk6IumvatGkAJCYmMmLECD799NMyL2P8+PElljmd5dZGp90Kaq33K6UmA18DDuAlrfVGpdRyYKrWenNFBVlRDMNg+JXn0TA2nOc/+okHF61h6u0J1I8KMzs0IUQpLViwgK1bt3Lw4EFuueUWzj77bJ5++mkyMzNJSUnhgQceoH///kyaNInu3bvTvXt3xowZQ9u2bfntt9+Ii4tj/vz5xMbGopRCa82CBQtISkpi79697N+/n+uvv567774bt9vNtGnT2LJlC02aNMEwDO655x4SEhIKjW337t1MnTqV5ORkIiIimDx5Mh07dmTZsmW89NJLWK1WWrRowdy5czlx4gT33XcfGRkZWCwWpkyZQufOnau4NgPKtTmstX4HeCffuKsKKbcHOLM866pIV/Y4k7joMB5/czMTF6xm+h09aN4o0uywhKg2Vm3ex1cb91XKsv/WvRV9u7Yq1zJcLhfLly8HYNy4ccyaNYs2bdqwbt06Zs+eTf/+/fOU37FjB7Nnz6Zdu3aMHTuWZcuWFejO0Vrz9ttvk5qaSv/+/Rk2bBiffvopp06dYsWKFRw4cIBrrrmm2LgmTJjA6NGjufzyy9m6dSvjx4/nv//9L/PmzeP9998nLi6Oxx9/nD///JOVK1dy2WWXMWrUKL777ju2bNliWiKos+eJdT8/nkfv6UVGpocJz6xmx97jZockhCiljh075gzPnTuXnTt3snDhQl599VXS09MLlI+Li6Ndu3YAtG3blpMnTxYok5CQgMPhIC4ujtjYWFJTU1mzZg3XXHMNhmHQvHlzevToUWRM6enp7Nu3j8svvxyAzp07ExMTw59//kmfPn246aabmDNnDldccQXnnXcePXr04JVXXuHf//43ycnJ3HLLLeWtltNWpzvIz2lVn7njejP9hfVMXrSWibdcSEL7piXPKEQt17dr+bfaK1NYWG537s0330xCQgIJCQn06NGD++67r0B5p9OZM2wYRqFnDhZWxmq1lvpAbGHL9Pv9eL1epkyZwo4dO/j222+ZMGECY8aMYdCgQXz++ed88803LF++nI8//phXX321VOuqaHV2jyBbs4aRzBnbmzPio5j92kaWr91tdkhCiFJKTk5mz549jB8/nksuuYSVK1fi9VbcDSd79uzJ8uXL8fv9JCUlsXHjxiJPsY2MjKRFixZ8+eWXAGzdupWjR4/Stm1bLr/8curXr8+dd97JoEGD+O2335gzZw5Lly7l2muvZerUqWzfvr3Q5VaFOr1HkC02ysnsu3sx563NLPpoG0dOnGLEVefJOdVCVHOxsbEMHTqUAQMGYLPZuOiii8jMzCQjI6NCln/DDTewY8cOrrnmGho1akSzZs3y7I3kN3fuXKZPn86CBQuw2+0sWLAAh8PBuHHjuO2223A6ncTFxfHYY4/hcrn497//zZIlS7BarTz++OMVEvPpMKrjxVXB0013r1y5skqfR+D1+li0ZBv/Xb+Xyy5swbgbLsBuq/M7TaKO2LJlCxdeeKHZYVQr33zzDX6/nz59+pCamsrgwYP56KOPiI2tHvcu27JlC2vWrOG2224jMjKSxMRE+vXrB3BW8CSdUpE9ghBWq4V7h3aiUf1w3vpiB8kpWTwwshsRYXazQxNCmKBNmzZMnDiRefPmAYEzlKpLEqhIkgjyMQyDv/dXNIwJZ8H7W7n/2e+ZfsdFxMWEmx2aEKKKtWzZksWLF5sdRqWTfo8i9OvWiqmjLiLpeDr3PbOafYfkHu1CiNpJEkExuqjGPHrPxXi9PiY++z2/7DpqdkhCCFHhJBGUoE2LWOaOu4T6UU4e+s86Vm/db3ZIQghRoSQRlEKTBhHMGdubti1jmfPmZj75dpfZIQkhRIWRRFBKUREOZt3Vk54dm/Ly0l948dOf8fmq36m3QghRVpIIysBhtzJxeDeu6d2apd/9yZy3NuNyV9xVjEKI0zNp0iSWLFlCUlISd9xxR6FllFLFLuOvv/7iwQcfBODnn39m8uTJ5Y5rwYIFLFiwoNzLqWxy+mgZWS0GdwxqT6PYcF5Z9ivJqVlM+Ud3IiPkEX9CmK1Jkya8+OKLpzXvgQMH+OuvvwDo0KEDHTp0qMjQqjVJBKfBMAyuvexs4mLCeHrxj0x8djXTR/WgcYMIs0MTokIcXvUNSStXVcqym/TrS+O+lxVbZsyYMVxzzTVcccUVAFx33XXMmjWLtLS0Qp89kC37QTarVq0iMTGRCRMmkJGRQadOnXLKJCUl8eCDD5Kamsrhw4e59tprGT9+PLNmzSIxMZEZM2bwf//3fzz77LO8+eabRT5jYNKkSURGRvLrr7+SlJTEvffey5AhQ4r8TF9//TXz5s3D5/PRsmVLZs6cScOGDXn88cdZs2YNFouF/v37M2bMGNatW8fcuXMBiImJ4cknn6RBgwanX+klkK6hcrjkghbMHN2D4yczmbDgO/7cX/DWtkKIssu+MyfAnj17yMrKol27drz11lvMmjWLjz/+mFmzZjF//vwil/Hwww9z3XXX8emnn9KlS5ec8Z999hlXX30177//PsuWLeP111/n+PHjTJkyhfbt2+c8HS3bhAkTGD58OMuWLeOBBx5g/PjxuFwuAA4dOsQ777zDokWLmDNnTpGxHDt2jKlTp7Jw4UKWLVtGly5dmDlzJvv37+e7775j6dKlLF68mD/++IOsrCyee+45pk+fzpIlS+jZs2el35BO9gjKqcPZDXl8TG+mv7iOSQu/54Fbu3GBamx2WEKUS+O+l5W41V6ZLr30UmbOnElaWhqfffYZAwcOBAI3dfv6669ZsWIFP/30U6HPHsi2ceNGnnzySQAGDhzIlClTALj99ttZv349L7/8Mjt37sTtdnPq1KlCl1HcMwYAevXqhWEYnHPOOSQnJxcZy7Zt2+jYsWPOvdP+/ve/88ILL9CkSROcTic33ngjffr04b777sPpdNKvXz/GjBlD//796devH7169SpjDZaN7BFUgDOaRvPE+Eto0iCCGS+tZ9Xmv8wOSYgazeFw0KdPH1atWsWKFSu4+uqrgcCzB7Zt20b79u256667SlxO9k01DcPAYgk0d4899hhvvvkmzZo14+6776Z+/fqFPksgdP7847JvdZ39DIOS7lSc/5kGfr8fj8eDzWbjgw8+YPz48SQnJ3PjjTeye/duRo4cyZtvvkmrVq2YO3cuixYtKvGzlockggoSFxPOY/dezPmt43h68Q98sPL3Ir9cQoiSDRo0iFdffZXY2FiaN29e5mcP9OzZk6VLlwLw5ZdfkpWVBcCaNWu4/fbbufLKK9m9ezdJSUn4fD6sVisejyfPMop7xkBZdOrUiZ9++onExEQA3nvvPRISEti+fTu33HIL3bp14/7776dNmzbs3r2b66+/nvT0dEaOHMnIkSOla6gmqRduZ/odPZj/7o+8sfw3jiSf4s5rO2K1yHMNhCirCy+8kNTUVG666Sag7M8emDp1KhMmTOC9996jffv21KtXD4A777yTiRMnEhYWRnx8PO3btycxMZHzzjuP1NRUJkyYwNChQ3OWU9QzBsqiYcOGzJw5kzFjxuB2u2nWrBmPPPIIjRs3pnPnzlx99dWEh4fTpUsXLrnkEsLDw5k0aRI2m42IiAhmzZp1mrVYOvI8gkrg8/l5Y/l2Pvr6DxLOj+e+Wy4kzCE5V1Rv8jyCmqeinkcgXUOVwGIxGHn1+dx1bQc2bj/ElEVrOZmWZXZYQghRKEkElWjAxa154NZu7D5wkgkLVnPwaNFnOAghhFkkEVSyHh2aMeuuXqRluJiw4Dt+33fC7JCEKFJ17CoWhct/JlJ5SCKoAued1YA5Y3sT5rDx4KI1bNp+yOyQhCjAZrPlXCglqr+MjIwKSwaSCKpIi8ZRzB3bm5aNI5n16kb+u36v2SEJkUfDhg3Zu3dvhW5piorn8/lIS0vj999/59ChQ/j9/pxrJE6XnMpShepHhzH7not57I1NPPvBVo4mn+LmK1SJF6MIURWaNm3KL7/8wg8//CDfyWrO5/Nx6NAh9u/fT1RUFGFhYeVaniSCKhbutPHQbQks/OAn3v1KcyQ5gzHXd8ZmlZ0zYS6LxUKHDh1Yv349GzZsKPdWpqhcfr+fyMhIBg8eLHsENZHNamHc3zvTqH44i7/UnEjJ4v4RXYkIs5sdmqjjDMOgR48edOnShczMTLPDEcWwWCxERERgtVrLvSxJBCYxDIObrziXhrHhLPzwJx5ctIZpt19E/ejy7eIJURGcTmfOfXRE7Sf7fia7POEMHrotgcTDady3YDWJh1PNDkkIUcdIIqgGup7XhEfv6YXL5WXigtVs333M7JCEEHWIJIJqom3L+swd15uoCAcPPb+WtdsOmB2SEKKOkERQjcTH1WPO2N6c1TyGx97YxLLVf5odkhCiDpBEUM3ERDqZdVdPureL54VPfubVZb/i88ll/0KIyiOJoBoKc9h4YGR3rup5Jku++YMn39mC21P0AziEEKI85PTRaspqMbjruo40qh/B659vJzk1iwdGdicyXK41EEJULNkjqMYMw2Bo37b86+YubN99jEnPruZocuEP2RZCiNNVrj0CpdTNwBTADszTWi/MN30QMAMwgN3AP7TWch/mMupzYUsaRIXxyGsbue+Z75h+Rw/ObBptdlhCiFritPcIlFLNgUeAi4HOwGilVLuQ6dHAImCA1roTsA2YXq5o67BO5zTi8TEX4/fD/c+uZtsfR8wOSQhRS5Sna6g/sEprfVxrnQ58CAwNmW4H7tVa7w++3wa0Ksf66ryzmsUwd1xv4mLCmfbCOr79IdHskIQQtUB5EkEz4GDI+4NAzpPmtdbHtNYfAyilwoFJwCflWJ8AGtePYM6Yizn3zAY88fYWlny9U54qJYQol/IkAgsQ2gIZQIEnWiilYoDPgZ+01q+XY30iKDLCwczRPbi4UzNe/Ww7L3zyM1651kAIcZrKkwgSgaYh7+OBPPdFUEo1BVYT6BYaVY51iXzsNisTbunK4Evb8Nn3u3n8jU1kueVaAyFE2ZUnEfwP6KeUaqSUigCGACuyJyqlrMAy4H2t9T+11rLJWsEsFoPbB7Zn1KD2rP/lIA89v5aUdHnmrBCibE47EQQPAk8Gvga2Au9orTcqpZYrpboCA4EuwFCl1Nbg30sVErXIY9AlbZg4vCt/JCYzccFqDh1LNzskIUQNUq7rCLTW7wDv5Bt3VXBwM3LBWpW5uFNz6keFMeuVDUxYsJppoy7i7BaxZoclhKgBpKGuRc5vHcecsb2x2yw8+Nz3/LDjsNkhCSFqAEkEtUzLJlHMHdub+Lh6zHh5Pf/buM/skIQQ1ZwkglooLiacx+69mI5nN2T+ez/y7ldarjUQQhRJEkEtFRFmZ+rtF9G3a0veXrGDhR/+hNdb4DIPIYSQ21DXZnabhX/eeAFxMWF8sHInx05mcv/wroQ55d8uhMglewS1nGEYjLiqHfcM6cgPO5J4YNEaklOzzA5LCFGNSCKoI67seRYPjuzOvkOpTFjwHQeOpJkdkhCimpBEUIcktG/K7Lt7kpHpYcKC1ei9x80OSQhRDUgiqGPUGQ2YO7Y39cLsPLhoLRt+OVjyTEKIWk0SQR3UrFEkc8b25oz4KGa/tpEv1u42OyQhhInk9JE6KjbKyey7e/H4m5t57qNtvPrZrzSIDicuJiz4V3A4NtKJ1SrbDkLUNpII6rAwp40p/+jOlxv38VdSKsdOnuLYyUx+3nWMEymZBZ5xYDEgNiqs8GQRHU6D4HBEmN2kTySEOB2SCOo4q9XClT3OLDDe5/NzMj2LYyczOX4yMydJHAsOHziazs+7jpF+yl1g3nCnrWCyiA6jQUjiiI0Kw2oxquATCiFKIolAFMpiMagfFUb9qLCQB5AWlJnl4XhKboI4djKTYym5w9v+OFr43oXFoH6UMzdZRIcF9yjydkmFy8VvQlQ6+ZWJcglz2mjWKJJmjSKLLOPz+TmZlpWbLPIljsTDaWzbeYT0TE+BeSPCbAW6nvIni5hIp+xdCFEOkghEpbNYDOpHh1E/OoyzWxb9jITMLE+evYnQZHH8ZCbbdh7heGoWvkL2LhpEOYmLKTpZxEWHya01hCiC/DJEtRHmtNG8USTNi9m78ObsXRSeLBIPp/LTziNkFLJ3US/Mluc4Rejxi+wkInsXoi6SRCBqFKvFoEF0GA2iw2jbsuhyp7I8BZLF8ZDjF38lpXKikL0La86xC9m7EHWHfJtFrRTutNGicRQtGkcVWcbr85OcmpmTLI7nO37xV1IqW38/wqmsgnsXVotBmMOK02HF6bAR5rAS5rAF3tsDw2FOa973IWWdDmveeRx5y8heiahKkghEnWW1GMGt/fBiy2VkugOJIiRJnMrykOnykuXykuny5LxmZnlITs3Kfe/ykuXy4Cvjc4HsNkswgQSThzOYNOzFJ5EwhxWn3YbTGVLGnreMw27FIolGhJBEIEQJIsLsRITZadmk6L2L4vj9fjxeH5kuL5lZXrLceZNI9nBWcDg7eWQFh7PLuNxe0k65OHoyMN6VnYTcXsr6ADqH3ZqbOPLspdhyEk2BPRln7ricMnnmCZS12ywYhiSamkQSgRCVzDAM7DYrdpuVqIiKX77f7yfLnZ1M8iUXd/aeSjC5uL05ySZ7TyZ0nuTUzNwywXlcnrI92c5ikKfLzGkP7IXYrBbsNgs2mwW7NffVbrPkTiusTHB8/jKh0wtdRsir7AEVTxKBEDWcYRjBrfHK+Tn7fP6chJIVssdSsGssJLm48753e3y4PV48Xj+ZGS48Hj9urzfwGhzv9nhxe/14PN4yd6WVxGIxik0WoYkpJ7kEh4tLMIUlrTzLCi1TxDJsVsP0Pahalwg8GRn8NutR/D4fjthY7LGxOOrHYo+NwR5bH0dsDPb6sThiY7E4HGaHK0S1Z7EYhDttVXqVt9eXN0F4PIHutTzj8r8Gk4vbE0gm7qLK5Ssfmoyygt1v+ROT2+PHk73sSnj2d/4EMfzK8+jfvVWFr6fI9VfZmqqIYbUS0aoVpxITyUhMxP3LL3hSC38al7VeRE6ysMcGkkMgScRgr18fe0wMjvr1scdEY7HLjdSEqCpWi4E1Zw+nev32Asd8Sp9kAgnMh9vry5ugitgr8nj9xV5LUxlqXSKwOp20ueuOPON8bjfukym4TpzAffIk7hMncCWfxH0iGVdyMu7kZDL27CE5ORlvekahy7VFRWKPybd3ERx21K8feI0NJA3Daq2KjyqEMEHgmE+gq6m2qHWJoDAWux1nwzicDeNKLOtzuQLJ4URyIFkkn8CdfBLXiUDCcCUnk/bHLlwnkvFlZhZcgGFgj44qsJcR2LsI7aqKxR4VJUlDCGG6OpEIysLicBDWuDFhjRuXWNabmRlIDieSA8kiX9JwJyeTsmMH7hPJ+FyuQlZmwR4Tnbd7KnsvIza4lxFMGrbISAxL7dkCEUJUH5IIysEaFoY1Pp6w+Phiy/n9frynTuVJGu7kgt1TpxITcZ1Ixu8peCWrYbVij4nJPYYR0jWV/yC4tV49089CEELUHJIIqoBhGNgiIrBFRBDerFmxZf1+P970jMDeRTHdU+l79uJOPonf6y24Pput2L2L0C4ra3i4JA0h6jhJBNWMYRjYIuthi6wHLYp5Igzg9/nwpKUH9i6K6J5yHT1G2h+7cJ9MAV/B094sDgf22BgsTicWhwOLzY5htwWG7TYsdgeG3R4ynD3NjmELLWfHsAeHHY7ANLs9Z3pgmj1k2IZhs0kSEqIakERQgxkWS+DAdHQUEa2KP+fY7/XiTk0L2bsIJo+TJ3EnJ+PLysLn9uBzu/G73XhSU/G53fhcbvyewGv2NJ/bTZnvaVDoBzCCCSSQZEKTRHYSyU0o9mBCyi5nDyaWYDm7DYutiHIOezDBBYeD043s9djtNfqgvd/vB58Pf/APvx+/LzjO7wu++vF7s6d5g68+8AVf/dnz+3PKk2d5weHQaTnLDClXYB2BeQ0DsFgwLBYMixUMA8MaeJ873hIYnz3OGihbYLwldL7Q8dZ8ZYxCygeXJRsgeUgiqCMMqxVHbAyO2JhyLyvQAHhzkkJOwnC78XlCht0hycMVmJYz7M6bWPKUc4ckH5cLT3oGfrcrb2Jye/C5XIUeTzktFkvu3outmIQRfA1URMmNZ3bjGNogF2y08zfI/nzL8OUrk7dhFqehqASRb5xhLWK8xQqWwsaHlLcWkriMQsZZg/dmylmnlUaXXUpk67OqrDokEYgyMwwDw2YDmw1rePF37qxsfp8Pv8cTTCae3IQRkih8Llcw0XjwufMPB8u53fg82cP5yrlc+DwevKdO4TuZEtgjgjyNBEbIVmierVorht3Iu4Wa3Rjk/PiN3K1kiyXQwBj5tnjzbRHnrtcIrCN7WSENSm4Dlm+9eZaZu3We57NkN05FrDe3YTPyLdOS+xlytr4tQGjyyreHUdJwduLz5h9fsGxOmfwJt8ThwsvnJvu86y+wl5RTxlvMevyFj/fnH+cnrGm8JAIhSsuwWDAcDrldiBDlICemCyFEHSeJQAgh6rjq2jVkBTh06JDZcQghRI0R0maW6TS46poImgIMGzbM7DiEEKImagrsKm3h6poINgG9gYNAwUtnhRBCFMZKIAlsKstMhr8iLgwSQghRY8nBYiGEqOMkEQghRB0niUAIIeo4SQRCCFHHSSIQQog6ThKBEELUcZIIhBCijquuF5SVilLqZmAKYAfmaa0X5pveGXgJiAa+A+7SWlfQDezLFdc04DbgRHDUi/nLVGJs0cBa4Gqt9Z5800ypr1LEZUp9Bdd7Q/Dt51rrifmmm/X9KikuM79fM4GhgB94WWv9VL7pZtVZSXGZWWdPAA211iPzjW8FvAU0BjQwTGudVhkx1Ng9AqVUc+AR4GKgMzBaKdUuX7G3gDFa63MAA7ijmsTVFbhRa905+FdVX7gE4HvgnCKKVHl9lTKuKq8vpVR/4HLgAgL/xwuVUtfmK2bG96s0cZn1/boU6At0DMYwViml8hUzo85KE5dZddYPuLWIyc8Bz2mtzwU2Aw9VVhw1NhEA/YFVWuvjWut04EMCGR8ApdQZQLjWen1w1GvA9WbHFdQVeFAptU0p9axSKqwK4oLAj+5e4ED+CSbWV7FxBZlRXweBf2utXVprN/AbkPM8UBPrq9i4gkz5fmmtvwX6BLfwGxPocUjPnm5WnZUUV1CV15lSqgGBjcbZhUyzA5cQaD+gkuuqJieCZgR+FNkOAi3KMN2UuJRSkcCPwASgCxBLJWb6UFrrUVrr1UVMNqu+io3LrPrSWv+a3WAppdoS6IpZHlLElPoqKS4zv1/B+NxKqRnAdmAlsD9kspnfsSLjMrHO/gNMJrc7KlRDICWk26xS66omJ4LA8+9yGYCvDNNNiUtrnaa1vkprvSP4T34SuKoK4iqJWfVVLLPrSyl1PvAVMEFrvTNkkqn1VVRcZtdXMIZpQCOgJXm7fkyts6LiMqPOlFKjgL+01iuLKJK/rqAS66omJ4JEgrerDoonb9dCSdNNiUsp1UopdVvIdANwV0FcJTGrvoplZn0ppXoR2HqcpLV+Pd9k0+qruLhMrq9zgweD0VpnAEsI9MtnM6XOSorLpDr7O3C5UmorMBMYqJR6OmT6YSBGKZX9XIGmVGJdlToRKKWilVK/KKXOLGRaZ6XUZqXU70qpl5RStuD4Vkqp75RSO5RSnwZ3wSrK/4B+SqmSfhQZAAAgAElEQVRGSqkIYAiwInui1novkBn80QAMB76owPWfVlzAKWCOUuospZRBoG/84yqIq1gm1ldJTKkvpVRL4BPgZq31u/mnm1VfJcWFud+v1sCLSimnUsoBDCJwEgBg6nes2Lgwoc601n/TWrfXWncGpgJLtdb/L2S6G1hNIGEAjKAS66pUiaAcZ5tU2lFvrfV+Av1rXwNbgXe01huVUsuVUl2DxYYBTyuldgCRwDMVtf7TjUtrfQS4E1hG4JQwg8CuqCnMrq+S4jKxvu4DwoCnlFJbg393VYP6KjYuM79fWuvlwOcE+tu3AGu11u+aXWclxVWdfpPBDemBwbf3EDjrcDuB57NMqaz1lup5BEqpl4DXgTeBy0LP8w6eCbBKa90m+L43MAO4AjgGNNBae4JbMt9qrVuXYn1OoBvyYBohhCiLnAfTaK2zSjtTqS4o01qPAih46i1Q9JkA5Tnq3Y3AbpEQQoiy603e7q9iVcSVxUWdCVCeo94HAd5++23i4+PLHaAQQtQFhw4dyn7W+8GSyoaqiERQ1JkAOUe9tdZeynbU2wsQHx9PixZVcpqxEELUJmXqUi/36aNFnQlQ1Ue9hRBCnJ7T3iNQSi0HpmqtNxM4E+BFFbhx2A/knglwD/C6UmoKsA+4qZzxCiGEqfx+Pz6XC7/bjc/lxufKCry63fhcrsCf240vy4XP7QqWcYXMU0yZ4PTm1w6iYc8eVfaZypQItNZnhgxfFTL8E9C9kPJ7gctOPzwhqobf78fv8eBze/C7XfjcHnxud+CH6wn84P3u4I/d7cEwAIsFw2LBsFoxLJbA++Bw9vjcMoWNK3w+LBYMwzC7Sqq1nP9XSOMZ+Av8n7xZWSENdcEGOf88gUY6tIw7OE/BYb+7fNeaGVYrht2O1enAsDuwOOxYHA4swWFbvXpYw8MrqKZKp0bfhlrUbH6fL9jYegI/tJxhd24jnDNcljKefONzG3B/cBk5ZUIa+WolO1nkJAoLhqWohGM5raRUMAFZcxOWxRqy3sLLF53UcpeRHZvf68vZci55qzhvg+zP3yBnBeajFKe+F8kwAo2vI9gQ2x1YnA4s9kCjbA0PwxYdFWykncEy9tx5Qocd9mCDnt2oB+ex2wud37BaS46vikkiqMP8Ph+etHTcKSm5u62FNrDuvFvIRTbCZWvA/d4KukTEYgn8yOx2DLst8Gqz5/74gj9AW2QkFrsNw27HYrMHXoNlDJstpKy9xDIAfq8Xv88X+PN6IXvY58Pv9QXfe/F7A+PwFVE+Z7ovd5nB1zzj8iy36DKErDO7nM/lDsRS6DJLiCWkTEWzOBzBOi68QbbVq1dIo5v3fejWdGENdWFb34bVKntdISQR1CJ+rxd3ahqelJO4T6bgTkkJeQ2OO3kST/b41NTT/nFnN4qG3Y7FbsNid+Q0wtnjs3/EueMLlsnfgBcsE1h2cY18ddzCqq0KJiBfEcnQmycZGlZbgUbbsNulMa4mJBFUYz6PB09KKu7shj2kUffkaeSD49LSitxdtkVFYo+Oxh4TQ1izZkSdd27wfTS2qGisYc5CGufQRj57OLBlbFhq8v0KxenK6SYStYokgirkc7vzNNzukynBBv1kvq33wHhPWhFPpTMMbFFR2GMCDXtEq5bYY87PbdijYwLTchr6KCw2+VcLIQonrUM5eLOy8m2Z53a/hDbo2dO9GRmFL8hiyWm07dHRRLY+K9igZ4/L17BHRkp3iBCiwkgiCPL7/fgyMwvtV/eENvIh032ZmYUuy7DZQrbOo4ls0hh7TExIYx+Tp6G31asnu9tCCNPU2kTg9/vxZmSEdMWkFHsQ1RM8c6YwFocjZOs8mvDmzXL62/N0xQSnWyMi5CCYEKLGqHWJwJOWzs8PTuHU/gP4PZ5Cy1jCwnK2zh2xsdQ7o1WwoQ/Zao/J7Y6xhIVJwy6EKDO/34/X58fj9eH1Bl59Pj8erx+vz5dnvNfnxxsc37ZVfZz2quv+rXWJwOJ00CChO36PJ0/Dbgtp4K1Op9lhCiHy8ftzG8jQxjG38Sw4PtB4+gLzeX14gu8LH5/d+GZPC5bLv2yvH4+v8HWGriN02b6ixvtO76K3IX3OZuTV51dwDRet9iUCu50zhsktjYQoD7/fj8vjIzPLQ5bLS6bLQ6bLm2848JoZHJeVfzgrWM7txe3x5W2o8zfK5Wg0T4dhgNViwWY1sFoMrNbgsNWCzWLBmn+8xYLNasHusGG1GtisgTK5ZS1Fjs9Zbki5osZnz3fumQ2qrC6gFiYCIeoKn8+Py52/IS7YYGdPO5VVSGNdRAOf5fJS1nbZ6bAS5rDidNgICw6HOWxERtix23Ib2EBjaQk2tEa+8SGNZEhjWbBRzh2fv/G1FVHeYsm7bpFLEoEQlcjr84dsOWdvKRe+VV2gXCENdmi5LFfZbtFhMQhppG05DXe4w0ZspJMwh40wpzU43hZs1HOHQ+fJHQ5Mc9itWKRxrbEkEQhRBK/PT0p6FsmpWZxMC7wmp7k4mZZFRqa72K6S7GG3p2y38LBZjTxb1NnDkeEO4mIKb4RzyjmLb6ztNrmrqSicJAJRp7jc3mCDHvg7mT0c8noyOC0l3VXoHTtsVoNwpz3Y8OY21rFRYXka4fzdJIVtjYc585azWeV6ElH1JBGIGs3v95Oe6cndYi+kQQ8ddyqr8FOKw52B7pHYKCdNG9bjvLPiAu8jHcRGhRET6SA2yklsVBj1wmyyZS1qFUkEotrxen2kpLuKbdBDu2o83oLdL4YBURHBxjvSSdsWscRGOYkJNvaxIa/RkQ7CHPJTEHWXfPtFlchye3O7YUK23k+GNvBpgffFdclkN+AxkU7OaBqd532exr2eA6t0swhRKpIIxGnJ7pJJTs0MNuiuwHCaK6dBL3WXTLDxbt4oknbZXTIhW+0xwe4Z6ZIRonJIIhA58nfJhDboJ0K6aE6W0CUTXc+Rs4We3SVToFsm0klMlLNKL6MXQhROEkEd91dSKgve38r+I2mkZhTVJWMJHjQNNOBnhnTJxEbmbeClS0aImkcSQR322+7jPPzKeqwWC706Nit4IDW4FS9dMkLUbpII6qh1Px/kibc20zA2nBmjexAfV8/skIQQJpFEUAd9sXY3zy/ZxtktY5l6+0XERMrdWIWoyyQR1CF+v5+3Vuzg/f/9TtfzmnD/8K6EOeUrIERdV6pWQCl1MzAFsAPztNYL802/Eng8+PZn4E6tdZpS6lJgCfBXcNqPWut/VEjkokw8Xh/PfrCVlZv+4m/dW3Hv0E5yUFcIAZQiESilmgOPABcCWcBapdTXWuvtwemxwOvAZVrr7UqpicBsYBzQFXhCa/1oZX0AUbJTWR4ee2MTP+w4zE2XK266XMnBXyFEjtJsEvYHVmmtj2ut04EPgaEh09sCe7MTA/AZMDg43A24XCm1TSm1VCnVsqICF6WTnJrFg4vWsFUfZsz1nbj5inMlCQgh8ihNImgGHAx5fxBoEfJ+J9BSKdUp+P4GID44nAws0Fp3BJYD75YvXFEWB46mMXHBavYdSmXyPxK44qIzzQ5JCFENlSYRWIDQy4wMIOeSUq11MjACeEEptQk4ALiC0+7SWi8JDj8PnK+Uiqmg2EUxft93gokLVpN2ys0jd/ek+/nxJc8khKiTSnOwOBHoHfI+nkBjD4BSygokaq0Tgu+7AbuUUhbgAeAxrXXoo5QKv+mMqDCbf0visTc2ERPpZMYdF9GicZTZIQkhqrHS7BH8D+inlGqklIoAhgArQqb7gS+VUs2VUgbwL+A9rbUPuDZYHqXUCGBD8DiDqCRfbdjLw69soHmjSJ4Y21uSgBCiRCUmAq31fmAy8DWwFXhHa71RKbVcKdU12ODfSSA5aOAEMDc4+63AP5VSvwL/AEZVwmcQBK4ReO8rzTPvb6Xj2Q159J5e1I8OMzssIUQNYPgLu8uYyZRSZwK7V65cSYsWLUoqXud5fX7+s2QbX6zbw2UXtmDcDRdgt8k1AkLUNYmJifTr1w/gLK31ntLOJ5eV1nCZLg9PvLWFDb8eYmjftoy46jw5PVQIUSaSCGqwlHQXD7+8Hr3vBHde24GrL25tdkhCiBpIEkENlXQ8g2kvrOPwiQzuH96NXp2amR2SEKKGkkRQA/25/yTTX1yHy+Nj5ugetG/T0OyQhBA1mCSCGuan34/wyGsbqRdm4/ExF3NGfLTZIQkhajhJBDXIN1v+Yv57P9K8USTT7+hBw9hws0MSQtQCkghqAL/fz8ff/MGrn22nfZs4Jv8jgchwu9lhCSFqCUkE1ZzP5+flpb+wdPWf9OrUjH/d1AWH3Wp2WEKIWkQSQTXmcnt5avEPrPnpAAN7t+b2ge2xWOQaASFExZJEUE2lnXLzyKsb+GXXMf5x9flce1kbuVBMCFEpJBFUQ0eTTzH9xXXsP5LGv4ddyGVd5DYbQojKI4mgmtl7KIXpL6wjPdPD9FE96HROI7NDEkLUcpIIqpFf/zzGw69swGGz8Ni9F9O6uTzDRwhR+SQRVBNrfjrAk+9soXH9CGaM7kGTBhFmhySEqCMkEVQDy1b/yYuf/oxqVZ+Hbr+I6HoOs0MSQtQhkghM5PP5eWP5dj76+g8Szo/nvlsuJMwh/xIhRNWSVsckbo+PZ97/kW+2JPJ/Pc7krms7YLXKw2SEEFVPEoEJMjLdPPr6Jrb+foRb/u9cbuh/jlwjIIQwjSSCKnYiJZPpL61nz8EUxv+9M/27n2F2SEKIOk4SQRVKPJzKtBfXczIti4duS6DreU3MDklUcz6fj127dpGSkmJ2KKIacTgcNG/enAYNGlTI8iQRVJEde48z86UNWCww++5enNOqvtkhiRrg4MGDGIbBBRdcgMUix5BEYOMgIyODXbt2AVRIMpBvVhXY8MtBJi9aS2S4nTlje0sSEKV29OhRWrZsKUlA5LBYLERGRtKmTRt27dpFWlpa+ZdZAXGJYvx3/R5mv7aRVvFRzBnbm2YNI80OSdQgHo8Hh0OuKxEFRUREYBgGn3zyCX6/v1zLkq6hSuL3+1n8pWbxl5oLz23M/SO6Ee6U6hZlJ2eUicJYLBYMw+DYsWOcOnWKiIjTvxuBtEyVwOv1sfDDn/hq4z76d2vFvdd3wibXCIgabsaMGfzwww+43W727dtHmzZtABgxYgRDhgwp1TLmz59P+/bt6devX5FlBg0axKefflohMdcFhmHg8/nKtQxJBBUsM8vD429uZvNvSfy9/zkM+79zZYtO1ArTpk0DIDExkREjRpxWYz1+/PgSy0gSqHqlSgRKqZuBKYAdmKe1Xphv+pXA48G3PwN3aq3TlFKxwNtAa+AIcIPW+lBFBV/dnEzLYubL6/njr2TuGdKRK3ueZXZIQlSJBQsWsHXrVg4ePMgtt9zC2WefzdNPP01mZiYpKSk88MAD9O/fn0mTJtG9e3e6d+/OmDFjaNu2Lb/99htxcXHMnz+f2NhYlFJorVmwYAFJSUns3buX/fv3c/3113P33XfjdruZNm0aW7ZsoUmTJhiGwT333ENCQkJOPB6Ph+nTp7Nz506OHj2KUoqnnnqKsLAwXnvtNRYvXozVaqVPnz5MmDCB/fv388ADD3D8+HHCwsKYNWsWkZGRjBgxglWrVuV8RoCxY8dy0UUX0b59e44cOcKHH37IjBkzSrWuu+++m379+rFy5UoiIyNJTExk9OjRLF++3JT/W7YSE4FSqjnwCHAhkAWsVUp9rbXeHpweC7wOXKa13q6UmgjMBsYBs4DVWusBSqnhwHzg75XzUcx16Fg6015Yx9HkU0y6tTs9OjQ1OyRRC63avI+vNu6rlGX/rXsr+nZtddrzu1yunAZt3LhxzJo1izZt2rBu3Tpmz55N//7985TfsWMHs2fPpl27dowdO5Zly5YxfPjwPGW01rz99tukpqbSv39/hg0bxqeffsqpU6dYsWIFBw4c4JprrikQy48//ojdbue9997D5/Nx66238u2339K0aVPeeecdPvroI8LDwxk1ahS//PILzzzzDFdccQXDhg3j22+/ZdGiRUyYMKHIz3rixAnuuOMOEhIS2LRpU6nXtWfPHi677DJWrFjB0KFD+eSTTxg8ePBp13lFKc0eQX9gldb6OIBS6kNgKDAzOL0tsDc7MQCfASsIJIIBwCXB8YuBhUopu9baXUHxVwt//JXMjJfW4/H6ePiunrQ7K87skISoch07dswZnjt3Ll9//TUrVqzgp59+Ij09vUD5uLg42rVrB0Dbtm05efJkgTIJCQk4HA7i4uKIjY0lNTWVNWvWcMMNN2AYBs2bN6dHjx4F5uvWrRuxsbG8/fbb/Pnnn+zZs4eMjAw2bdpEnz59iIqKAuC1114DYNOmTTz11FMAXHrppVx66aUkJiYW+3k7dep0WusaMmQICxYsYOjQoXz22We8/vrrxa6nKpQmETQDDoa8Pwh0D3m/E2iplOqktf4JuAGIzz+v1tqjlEoBGgEHyht4dfHDjsM8+vpGouo5mH1PL1o2iTI7JFGL9e1avq32yhQWFpYzfPPNN5OQkEBCQgI9evTgvvvuK1De6XTmDBuGUegpkIWVsVqtJR4cXblyJc888wwjRozguuuu48SJE/j9fmw2W55jdklJSYSHh2Oz5TaFfr+fXbt2ER4enicmj8eTp1z25y3rurp168bhw4f58ssvadGiBU2amH+HgdKcymIBQv9DBpDzX9BaJwMjgBeUUpsINPKukLIUNW9Nt2rzPma+vJ6mDesxd2xvSQJCAMnJyezZs4fx48dzySWXsHLlSrxeb4Utv2fPnixfvhy/309SUhIbN24scELGunXruPLKKxkyZAjR0dFs2LABr9dL165d+fbbb0lPT8fj8fDvf/+bX375ha5du/L5558DsHbtWh566CGio6NJTk7m+PHjuFwuVq9eXWg8ZV2XYRgMHjyYWbNmcd1111VYvZRHafYIEoHeIe/jCdmiV0pZgUStdULwfTdgV3Dy/mD5RKWUDYgCjlVA3Kby+/18uGonbyz/jY5nN+TBkd2pF243OywhqoXY2FiGDh3KgAEDsNlsXHTRRWRmZpKRkVEhy7/hhhvYsWMH11xzDY0aNaJZs2Z59kYArr/+eu677z4+//xz7HY7Xbp0ITExkeuvv55bbrmFG2+8EZ/Px9/+9jd69uzJWWedxZQpU3jnnXcIDw9n1qxZREVFMWrUKIYOHUp8fDwdOnQoNJ6yrgtgwIABvPLKKwWOm5jFKOmKtODB4u8JdAelA2uB0VrrjcHpFmAfkEAgQbwD/Ky1nq2UWgjsDw4PA27WWg8oKSil1JnA7pUrV9KiRYvT/nCVwevz88LH21i+dg+XXNCcf97YBbtNrhEQlWPLli1ceOGFZodRrXzzzTf4/X769OlDamoqgwcP5qOPPiI2Ntbs0ErF5/OxePFidu/ezZQpU8q1rC1btrBmzRpuu+22nLOQgtdonKW13lPa5ZS4R6C13q+Umgx8DTiAl7TWG5VSy4GpWuvNSqk7CRwgdgL/A+YGZ38IeE0p9SuQDAwrw2esdrLcXp58ewvrfj7ItZedzcgB7bBY5BoBIapSmzZtmDhxIvPmzQMCZyjVlCQAMGbMGA4ePMjLL79sdig5SnUdgdb6HQJb+qHjrgoZ/hz4vJD5jgMDyxljtZCa4eLhlzewY+9xRg1qz6BL2pgdkhB1UsuWLVm8eLHZYZy25557zuwQCpAri0vh8IkMpr+4joNHM5hwS1d6d25udkhCCFFhJBGUYPeBk0x/cT1ZLg8zR/egw9kNzQ5JCCEqlCSCYmz74wiPvLqRcKeNx8b05sym0WaHJIQQFU4SQRFW/7ifpxb/QNOG9ZhxRw8a1Q83OyQhhKgUkggK8cm3u3h56S+0O6sBU25LICpCHgwihKi95AT4ED6fn5eX/sLLS3+hR4emPHxnT0kCQgTddNNNOVffZsvIyCAhIYHjx48XOd/w4cPZsGEDP//8M5MnTy4wPTExkb59+xa77m3btjF3buCs9JUrVzJ//vzT+ASiKLJHEOT2eJm3+Ee+27qfAb3O4o7BHbDKNQJC5BgyZAjLli1jwIDca0K//PJLEhISSvUA9Q4dOhR5dW5J/vjjD44dC9yUoF+/fsU+2EaUnSQCIP2Um9mvbWTbH0e5dUA7hvQ5Wx4mI0Q+V155JXPmzCE5OTnnAq6lS5dy6623AvDFF1/w6quvkpmZicvlYvbs2XTp0iVn/g0bNvDss8/y5ptvsn379py9g3PPPTenzO+//87DDz9MRkYGx48fZ/To0QwYMIBnnnmGjIwMFi1aRJMmTdi4cSOPPfYYW7du5ZFHHiErK4v69eszc+ZMzjjjDIYPH06HDh3YsmULx48fZ8qUKVx66aV5Pk9h67rppptITk5m8uTJ/PnnnzgcDiZNmkSPHj1YtmwZixYtwjAMOnTowMMPP8zzzz8PBJ5RANC3b1/eeOMNNm7cyMcff0xycjJ9+vTh6quvLvW6EhMTWb9+PU8++SQQeA6C0+lk9OjRlfSflUTAsZOnmP7iev5KSuX/3dSFvl1bmh2SEEU6vOobklauqpRlN+nXl8Z9Lytyer169ejXrx8rVqzgxhtvJCkpid27d3PxxRfj8/l49913ef7552nQoAEffvghL7zwQk5Dmd/999/PpEmT6NWrFwsXLmTDhg0AfPDBB9xzzz306NGDv/76i4EDB3LTTTcxbtw4Nm7cyN13382SJUuAwPMP/vWvfzFv3jw6duzIF198wb/+9S8++ugjANxuN++99x6rVq1i/vz5BRJBUeuaP38+rVq1YuHChWitmTp1Kq1bt+bRRx9lyZIlxMfHM2HCBL799tti6zMpKYnly5djs9l45JFHSr2uV155haeffpq0tDQiIyP57LPPeOONN0r5Xzw9dfoYwV9JqUxYsJqk4+lMHXWRJAEhSnDdddfx2WefAbBs2TIGDhyI1WrFYrGwcOFCvv/+e+bPn8/HH39c6DMIAI4fP87hw4fp1atXzjKzTZo0iaysLP7zn/8wb968Ym9Ut2fPHqKjo3Oeg3DllVeyb98+UlNTAejdO3CvzLZt25KcnFxg/qLWtWnTJgYNGgSAUor33nuPH3/8kS5duhAfH7jD/ty5c0u8YVy7du1ybltdlnXVq1ePSy+9lK+++orNmzfTsmXLSr9VdZ3dI/j1z2PMemUDNpuF2fdczNktas69SkTd1bjvZcVutVe2bt26ceTIEQ4ePMjSpUt59tlnAUhPT2fo0KEMHDiQbt26oZTi7bffLnQZ+Z89YLVac4b/+c9/Eh0dTZ8+fbjqqqtykk5hCnsmgd/vz7nldfazDIrq5i1qXfmfI7Br164C47IPjud/cLzbnfvMrdA7opZlXWeddRZDhgxh0aJFtGjRokpuVV0n9wjW/XyAqf9ZS3Q9B3PH9pYkIEQZDB48mEWLFhETE0OrVoGH5OzZswfDMLjrrrtISEjgq6++KvIZBPXr16dZs2Z88803AHka+zVr1jBu3Dj69+/Pd999B4DX68VqteLxePIsp3Xr1iQnJ7Nt2zYAli9fTrNmzUp9A7qi1hX6bIJdu3Zxxx130KFDB7Zu3cqRI0cAmD17NitXrqR+/fr88ccfQODMpuzp5VmXYRh07dqVQ4cOsWHDhiq5VXWd2yNYvnY3/1myjbYt6/PQ7QnERDpLnkkIkeO6666jb9++PPLIIznjzj33XM477zyuvPJKDMPg4osvZsuWLUUuY+7cuTzwwAPMmzePzp0754wfO3YsN998M06nk3PPPZfmzZuTmJhIx44defbZZ3niiSdo3bo1AA6Hg6effpqHH36YU6dOERMTw9NPP13qz1HUusaNG8eUKVMYOHAgNpuNOXPm0KRJEyZPnsztt9+Oz+ejc+fOXHfddaSkpPDf//6Xq666ivPPPz/n0ZvlWVf2HsLf/vY3kpOTcTgq/xT2Ep9HYIbKeB6B3+/nzS9+44OVO+nWrgkTh3clzFHn8qCoYeR5BHWP3+/H7Xbzj3/8gwcffJDzzz+/yLIV9TyCOtE15PH6mPfuj3ywcidXXHQGk0d2lyQghKiWjhw5Qq9evejUqVOxSaAi1frW8FSWh8de38QP+jA3X6648XIl1wgIIaqtxo0bs2nTpipdZ61OBCdSM5n50nr+3H+SMdd34oqLzjQ7JCGEqHZqbSI4cCSNaS+u43hKFpNvS6B7u3izQxLitPh8PiyWOtGLK8qgsNNnT1etTAS/7zvBjJfW4/fD7Lt7os4o+T4oQlRHERERJCUl0aRJE0kGAggcTHa5XOzZs4fMzEz8fn+5u7trXSJIy3AxedEaYiKdzBjdg+aNIs0OSYjT1qZNG3bu3Mn+/fvl2JbI4ff7SU5OZu/evTidzjwXr52OWpcIwp02Rg5oR8+OzagfXb7KEcJsDoeD888/n127drF8+XIAquMp38IcNpuNwYMH57k6+7SWU0HxVBtWq4UBF7c2OwwhKlSbNm0YOXIkqampFdo3LGoui8VCTEwM9erVK/eyal0iEKK2ioqKIioqyuwwRC1UXROBFeDQoUNmxyGEEDVGSJtZpr6i6poImgIMGzbM7DiEEKImagrsKm3h6poINgG9gYNA4bcwFEIIkZ+VQBIo06XJ1fKmc0IIIaqOXKEihBB1nCQCIYSo4yQRCCFEHSeJQAgh6jhJBEIIUcdJIhBCiDpOEoEQQtRx1fWCslJRSt0MTAHswDyt9cJ80zsDLwHRwHfAXVprTzWIaxpwG3AiOOrF/GUqMbZoYC1wdf6HW5tVX6WIy5T6Cq73huDbz7XWE/NNN+v7VVJcZn6/ZgJDAT/wstb6qXzTzaqzkuIys86eABpqrUfmG98KeAtoDGhgmNY6rTJiqLF7BEqp5sAjwMVAZ2C0UqpdvmJvAWO01ucABnBHNYmrK3Cj1rpz8K+qvnAJwPfAOUUUqVbS3dcAAANPSURBVPL6KmVcVV5fSqn+wOXABQT+jxcqpa7NV8yM71dp4jLr+3Up0BfoGIxhrFJK5StmRp2VJi6z6qwfcGsRk58DntNanwtsBh6qrDhqbCIA+gOrtNbHtdbpwIcEMj4ASqkzgHCt9frgqNeA682OK6gr8KBSaptS6lmlVFU9OOEO4F7gQP4JJtZXsXEFmVFfB4F/a61dWms38BvQKnuiifVVbFxBpny/tNbfAn2CW/iNCfQ4pGdPN6vOSoorqMrrTCnVgMBG4+xCptmBSwi0H1DJdVWTE0EzAj+KbAeBFmWYbkpcSqlI4EdgAtAFiKUSM30orfUorfXqIiabVV/FxmVWfWmtf81usJRSbQl0xSwPKWJKfZUUl5nfr2B8bqXUDGA7sBLYHzLZzO9YkXGZWGf/ASaT2x0VqiGQEtJtVql1VZMTgYVAf182A/CVYbopcWmt07TWV2mtdwT/yU8CV1VBXCUxq76KZXZ9KaXOB74CJmitd4ZMMrW+iorL7PoKxjANaAS0JG/Xj6l1VlRcZtSZUmoU8JfWemURRfLXFVRiXdXkRJBI8HbVQfHk7VooabopcSmlWimlbguZbgDuKoirJGbVV7HMrC+lVC8CW4+TtNav55tsWn0VF5fJ9XVu8GAwWusMYAmBfvlsptRZSXGZVGd/By5XSm0FZgIDlVJPh0w/DMQopbKfK9CUSqyrmpwI/gf0U0o1UkpFAEOAFdkTtdZ7gczgjwZgOPCF2XEBp4A5SqmzlFIGgb7xj6sgrmKZWF8lMaW+lFItgU+Am7XW7+afblZ9lRQX5n6/WgMvKqWcSikHMIjASQCAqd+xYuPChDrTWv9Na91ea90ZmAos1Vr/v5DpbmA1gYQBMIJKrKsamwi01vsJ9K99DWwF3tFab1RKLVdKdQ0WGwY8rZTaAUQCz5gdl9b6CHAnsIzAKWEGgV1RU5hdXyXFZWJ93QeEAU8ppbYG/+6qBvVVbFxmfr+01suBzwn0t28B1mqt3zW7zkqKqzr9JpVSLymlBgbf3kPgrMPtBJ7PMqWy1ivPIxBCiDquxu4RCCGEqBiSCIQQoo6TRCCEEHWcJAIhhKjjJBEIIUQdJ4lACCHqOEkEQghRx0kiEEKIOu7/A2dkMbLmJe8pAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Plot the loss and accuracy curves for training and validation \n",
    "fig, ax = plt.subplots(2,1)\n",
    "ax[0].plot(history.history['loss'], color='b', label=\"Training loss\")\n",
    "ax[0].plot(history.history['val_loss'], color='r', label=\"validation loss\",axes =ax[0])\n",
    "legend = ax[0].legend(loc='best', shadow=True)\n",
    "`\n",
    "ax[1].plot(history.history['acc'], color='b', label=\"Training accuracy\")\n",
    "ax[1].plot(history.history['val_acc'], color='r',label=\"Validation accuracy\")\n",
    "legend = ax[1].legend(loc='best', shadow=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##  混淆矩阵"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "目的：解释误分类的情况 —— 哪个类别容易被误分类成哪几种类别"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "Y_pred = model.predict(X_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "Y_pred_class = np.argmax(Y_pred, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "Y_true = np.argmax(Y_val, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "confusion_mtx = confusion_matrix(Y_true, Y_pred_class)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_confusion_matrix(cm, classes,\n",
    "                          normalize=False,\n",
    "                          title='Confusion matrix',\n",
    "                          cmap=plt.cm.Blues):\n",
    "    \"\"\"\n",
    "    This function prints and plots the confusion matrix.\n",
    "    Normalization can be applied by setting `normalize=True`.\n",
    "    \"\"\"\n",
    "    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n",
    "    plt.title(title)\n",
    "    plt.colorbar()\n",
    "    tick_marks = np.arange(len(classes))\n",
    "    plt.xticks(tick_marks, classes, rotation=45)\n",
    "    plt.yticks(tick_marks, classes)\n",
    "\n",
    "    if normalize:\n",
    "        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n",
    "\n",
    "    thresh = cm.max() / 2.\n",
    "    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n",
    "        plt.text(j, i, cm[i, j],\n",
    "                 horizontalalignment=\"center\",\n",
    "                 color=\"white\" if cm[i, j] > thresh else \"black\")\n",
    "\n",
    "    plt.tight_layout()\n",
    "    plt.ylabel('True label')\n",
    "    plt.xlabel('Predicted label')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAU8AAAEmCAYAAADiNhJgAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAIABJREFUeJzt3Xd8FHX6wPHPbiihKqEXFWmPiCIKigUEqQI2UI4TVFDEBvb6OxsoKHZFPfVUTk8Oe1cQASsWPFQQ23PoYUGK9CaBQPL7YyawxGR3szuTnUmet699yc7OPvPNZPPsd8r3+0QKCgowxhhTOtFMN8AYY8LIkqcxxqTAkqcxxqTAkqcxxqTAkqcxxqTAkqcxxqSgUqYbYEpPRLKAi4GhOL/DKsDrwA2qujWNmC8BbYFJqvpAKd/fCbhGVU9JZfteE5E9gJdVtUcJr88HuqvqurJtmSkvLHmG00NAHaCnqq4XkRrAv4HHgNNTjNkU6AvUUNUdpX2zqs4DApE4XXWAw0p6UVU7lGFbTDkUsZvkw0VEmgPfAI1VdUPM8kbAUar6otvrehDoABQA04G/qep2EckFJgJ9gMbA7cAUYC4gwELgZOAHoL6qrnLjFwD1gVzgn0BrIB/4HDgXOBp4QFUPKO32VfWhYn7OXOBuoBdQExgLDAYOBJYCx6vqZhE5y91+FSAHmKiqD4nIu26bFgIdgT+AV4GDgGHAf9yfZzTOl0ZX9/kXwDBVfTf534qpiOycZ/h0BL6JTZwAqrpcVV90n04CVuMkmk44CeMK97WqwCpVPRKnp3gPkAf0B7aoagdV/THO9gcCtdye26HushZF1inV9kUku5jtVAWWq+phwJM4vepLgP2BPYATRaQmMAror6oHA0NwvgwAzoz5eXbgntpQVXF7yYXGuz//lcBTOF8AljhNQpY8wyefxL+3fjhJoMA9B/qwu6zQq+7/v8BJUjVKsf05QDsReQ+4BrhXVX/wafuFXwY/AgtV9TdVzQcWAzmqugk4DhggIjcD1+L0UkvyYdEFbmIdBlwNRIBb47zfmJ0seYbPXKCtiNSKXSgiTUXkTRGphvN7jT0fEwUqxzzfAqCqhetESthWxI1dpXCBqi4GWuEkmdrALBE5vsj7vNp+7MWvvKIvikgzYD6wD05Sv66EOIU2lbB8H7dNLXHOlRqTkCXPkFHVpTgXhyaLSG0A9/9/B1ar6hZgBjBGRCIiUhU4B5hZyk2txDnkBueqPu62zsc55/m2ql7tbuuQIu/1YvvJ6OS2czzwNk4vtPDOge1AloiUlJhx190TZ3+OAJ4GHvehnaYcsuQZThcA3wIfu7fczHWfn+2+fhHQAOdiyUJAgQml3MZFwIMi8gXO7UvL3OX/ArKAb0Xkc5zzj5OKeW+620/G28ASN/53wN44ybSV297PgG9EpG6cGI8Cb6jq2zgXpVqIyAU+tNWUM3a13RhjUmA9T2OMSYElT2OMSYElT2OMSUHghme6V2cPxTnhX+phgsYY32ThjAr7T6pzKBRHRHJwbntLxgZVXePVttMRuOSJkzj/dDOzMSYwuuLcV5s2EcnZQaXVWWxP9i1rRaRVEBJoEJPnMoCVLU8jv0qyX0bJ+/y+kz2PaUxFsGL5cs48Yxjsum3NC7Wz2M6Kqp3YHilulO4ulQpyabh1Xh2cXqolz2LsAMivUpsdVff0PHjTps08j2lMBeP56bTt0ersiFaLv1J+sC7RBDF5GmMqmggQiTsYrORBvBliydMYk3mRqPNItE6AWPI0xmReNMt5xJXo9bIVrFQeR73a2Xz34Cm0abLrItKtZxzKWb3a7L5erap8ee9AqlZO/UfLz8/nwgvOo1uXI+jTszs//lB0xrXU+Rnb7/gW22L7J+Ictsd7BOy4PRTJs1JWhEmjjiB3m3M7Q71aVXnxmp7077jXbuv1PKgJr1zbmwZ7xL9ql8hrr75Cbm4u78/5hJsnTOSaqy5PK15ZxfY7vsW22L6JRHYdupf4sORZahNO68Tjs5Rla7cAUCO7Mre+sIBnPtx9wvP8/AJOGD+TtZu2pbW9jz+aQ+++xwLQ+fDD+fzzeQneEYzYfse32BbbN4l6nTt7n8Hha/IUkaEi8q2ILBKR0anEGNatJas2bmX2gqU7l/28chPzflj1p3XfXbiMNZvSH/iwccMG9thjj53Ps7Ky2L496Zt4Mxbb7/gW22L7JmGvM4kLSmXMt9aISFOcORy74BQCO0dE9i9tnNOPaU2PAxsz7Ya+HNg8h0dGd037sDyRWrVrs3Hjxp3P8/PzqVTJm2trfsb2O77Ftti+sZ7nbnoB76jqGlXdDLxACqVpjx37Fv3GzaD/TTNY+NMazn3wQ35fn+t5Y2MdceRRzJg+DYC5n37KAQccGIrYfse32BbbNyHsefr5VdOE3YdxLSNOHe0gOfGkgbwzaybdux5JQUEB/3jsn6GI7Xd8i22xfZNMzzJgPU/fZpIXkWuBbFW93n0+CuioqucleF9zYPGKthf4Mjxz5b+Hex7TmIrgt9+W0L9PT4B9VfUnL2IW/r3/Vu94dmTFK3wKWTs20XTV655uPx1+9jyX4My+UqgRsLSEdY0xFVk0C7IS3ARfEKyb5P1MnrOAsSJSH9gMnIxTRdEYY3YXwuGZvrVGVX8DrgXexamtPVVVP/Nre8aYEAvh1XZf701Q1anAVD+3YYwpBwpHGCVaJ0BsYhBjTOaF8Gq7JU9jTOaF8JynJU9jTAAkc07Tep7GGLM763kaY0wKrAyHdz6/72RfirXVOXSM5zELrfnsft9iA0QCdsLcGM8kM5N8wpnmy1Zgk6cxpgKxw3ZjjEmBJU9jjEmBnfM0xphUJDNfp/U8jTFmdyEcYRSsVF4KXpZOrV+nJoum30yb5g1p36Yp7z95ObMnX8rDNw7b7Qp3vTo1WfjqDVStkvp3Tl5eHiNHnEGvY46m65GdeeP111KOVRy/S8p+NncufXp29zQmhLfMblmU8PVjnweu9HAIZ5IPVmtKwavSqZUqRXngulPZsjUPgGvP7c8tj06n51n3ULVKJfp1bQdAryPa8vrfR9Mgp1Za7X566hRy6uYw690PeOX1aVx2yYVpxSvKz5Kyd915Oxeceza5ud6XQQlrmV2/S/j6tc+DWXo4XLMqhTZ5elU6deKlA3n0hTksW7kegPn6K3Vq1wCgZo1s8rbvAJyyxgPOe4C1G/5Iq92DTh7MDWNv3vnc66JbfpaUbdGiJc88/5Jn8WKFtcyu3yV8/drnQSs9HIlEknoEie/JU0Rqi8jX7nT7nvGidOppx3dm5dpNzPrku53LfvxlJXdddQrzX7qOhjm1+GDeIgDemfs9a9ZvTrvdNWvWpFatWmzcuJFhfx3MjTGJ1At+lpQdOOhkKleu7EmsosJaZtfvEr5+7fOglR52OpaJkmfGmlcsXy8YiUhn4FGgjdexvSidOvykIygoKKBH5/1oL015/ObTaS/NOPyvE/nuf8s59y9HM/GyQVw68TlP277k118ZMngQ55x3PkNOHepp7MCVlE1SWMvs2v72RiQaIRKNnx0TvV7W/O55jgJG40PtIi9Kp/YeeS99zr6PvqPu4yv9jZHXP8XiJavYuNk5v7Rs5Trq1K7uabtXrFjB8f37Mv6WiQwfcZansSGAJWWTFNYyu7a/vREhicP2gN3o6fdM8mcDiIjnsf0qnXrBTVP518Qz2b4jn215O7jgJm8nwr/jtltYu24tE28Zz8RbxgPwyuvTqFatmifxA1dSNklhLbNr+9sbyZzTDNo5T99KD8cSkZ+A7smUCy0sRTrt7dk2MUgRQfvwmIrFz9LDK9uNSVhqPGvrOup/80Cpty8idwL1VHWEiHQAHgNqAx8A56nqdhHZG5gCNAAUGKaqm+LFDe3VdmNMORJJ8lFKItITGB6zaAowRlXbuBFHucv/DvxdVfcD5gHXJ4od/DPbxpjyL5lbkXa93qyYU4HrVHVd7AIRyQEmALcAB4nIPkA1Vf3UXeUJYJyIPAYcDZwUs/x94Op4zbGepzEm40p5n+eHwOIij0uKCfsITvnzte7zJsCymNeXAc2AesAGVd1eZHlcZdLzVNXmZbEdY0w4lfKCUVdgSZGXi/Y6zwZ+VdXZIjLCXRwFYi/yRID8YpbjLo/LDtuNMRlXeJN8onVcS5K4YDQEaCwi84EcoCZOgmwcs04jnNsofwf2EJEsVd3hrpPw9ko7bDfGZFwkEtl5o3yJj1LcbaKqvVX1AFXtANwAvKaqZwK5InKUu9rpwHRVzcM5FTDEXX4GMD3RNqznaYzJuDK8z3MY8KiI1Aa+ACa5yy8AnhSR64BfgFMTBbLkaYzJOD+Tp6o+gXMFHVVdABxWzDo/A91LE9eSpzEm85K5jzNgY0QqXPJc+58HfIvdaMQU32IDLH/iNF/jm7Ll5+i+sI1GC+PwzAqXPI0xAVS6m+QDwZKnMSbjIknMFG89T2OMKSJCEskzYCc9LXkaYzLPLhgZY0zpRSIkcdheNm1JVmhHGIWpnGy92lX5+r6BtG5cm30b1mT69X2Ydn0f7hpx2G4fiH0b1uTjW48LVNstduZi+1mmOmilh6PRaFKPIAlWa0ohLOVkK2VFuPeszuRuc6pw3jKsExNeWED/m98mEoEBHfcCYMhR+zJ5dFfq1qoamLZb7MzG9rNMdfBKDyf5CJDQJs+wlJMdP7Qjk2cvYvm6LQAc1DyHOd+tAGDWgqV0a9cIgHWbtzFgwttptjw8+8ViJ+ZnmWorPZw+X5OniNwoIt+4j9u9jB2GcrJDu7Zg1YZc3lm4awrB2N//ptw8alevAsCM+b/xx9YdqTfaFYb9YrGT42eZ6uCVHg5f8vTtgpGI9AL6AAfjTAX1logMVNWXvYgfhnKyp3VrSUEBdD+gMQfuXYeHzzuS+rWzd75eM7sy6zdv86TNhcKwXyx28vwqUx200sOQTHIMVvL0s+e5DLhcVbe5Uz59B+ztVfAwlJPtP34mAybM5LgJM1n4y1rOe/hjZi1YSpe2DQHodVATPtHfPWs3hGO/WOzk+FmmOnClh63nuYuqflP4bxFpDfwFOKrkd5ROWMvJXjv1cyaNPJzKlaL8d+l6Xv3sF89iQ3j3i8X+Mz/LVAet9HAY7/P0vfSwiLQD3gRuVNUnk1i/OT6WHvaTTQxiSiNsE4P4WXo475jroXpO/JX/WEPld2/2dPvp8PUkhztj84vAJar6jJ/bMsaEVzI3yQet5+nnBaO9gFeAIar6jl/bMcaEXzQagWiC7Jjo9TLmZ8/zCiAbuDumxvLDqvqwj9s0xoRQJITnPP28YHQxcLFf8Y0x5UcysyoFLXvaxCDGmMxLIncWBCt3WvI0xmRe1C0vHE9BNEJ+GbUnGZY8jTEZl8RE8kE7arfkaYzJvKRGEFWUEUbGGJMs63kaY0wKrOdZwfk9fLLe0Cd8i71q6gjfYodtGKLJhMTJsyBgXU9LnsaYjItGI84oo7grWfI0xpjdJHPOM2gHGZY8jTEZ5yTPcFXPtORpjMm4MPY8Q1sALqzlZL2OXb92Nt//fTBtmuyqRzNx+KGM7L1zMhZGD9ifdycM4N0JA/i/Uw4KTNtj7dixg3NHnUWPbl3o3aMb//vxR89ih+n3GSus+yQVYZxJPrTJM6zlZL0uazzpnCPYss0p3FWvVlVe+r9e9HfLGQM0b1CTIV1a0PO6afS47k16tG9Cu73rZLztRb35xusAvPP+HK6/cRxXXxnMfV6WscO6T1JR2PNM9AiS0B62h7WcrJexbzn9UB6bqVxxUnsAamRX5pbn59Pn4F0z8C9ZvZmBt8wk371dqHKlKFvzUqvS6ed+OeHEk+g/4DgAfvn5Zxo0bOBZ7LD8PosK6z5JjRWA242I3CQi37qlhy/zMnZYy8l6FXtYt1as2pDL7AVLdy77eeUm5v2warf1tu8oYPXGrQBMOL0TXy1eww/LNmS07SWpVKkSo84aweWXXsTAQad4FjcMv8+ShHGfpCKMPU/fkqeIdAN6AO2BTsCFEjMrcrrCWk7Wq9hnHNOKHu2bMP3GYzmweQ7/GNOFBnsUXxisauUsJl90NLWyK3PJY59mvO3xPDr5CRZ8o4w+/xw2b97sScww/D7jCds+SYWd84yhqu8Dx6jqdqABzikCb37zhLecrFex+459i2PHvkW/cW+x8Kc1nPPAHH5fv6XYdZ+9sgcLf17DRY9+svPwPZNtL87UKU9xx223AlC9enWi0ShZWVmexA7D77M4Yd0nqSi8ST7RI0h8/apR1TwRGYdTkuN54DevYoe1nGxZl3w9/tC96bJ/I6pWzqJPB+dc6I1TP+ezRStLHcvX/TJwEOeefRa9e3QjLy+P2++8h+zsbG9ih/T3GdZ9koow3qrke+lhABGpDrwOPKuq/0iwbnNCWnrYbza2/c+CdihXGmHbL36WHq576m1k1aoXd90dG1ex+umrPd1+Ovw857mfiHQAUNU/gJdwzn8aY8xuwnjO08/D9hbAOBHpAhQAJwKTfdyeMSbEvM6NInITcApO/nlcVe8WkV7A3UA1nCPh69x1OwCPAbWBD4Dz3Os1JfLzgtE04E3gS+Bz4GNVfcav7RljwsvrnmcJd/schNOBOxFoCxwqIv3ct0wBxqhqG5wbSkcl2obfF4zGAmP93IYxJvxKecGoWTF3Pa5T1XWFT1T1fRE5RlW3i0hTnFy3J7BIVRcDiMgUYLCIfAtUU9XC+/ieAMYBD8VrT2iHZxpjyo/CWZXiP3au/iGwuMjjkqIxY+72+RaYDTQBlsWssgxoFmd5XJY8jTEZV8oRRl2BfYs87i0urqreCNQH9gLa4Jz/3LlZIB8nDxa3PK7Qjm03xpQfWdEIWUnUbXctSXSrkojsB2Sr6nxV/UNEXsK5eBQ7sUMjYCmwBGhczPK4rOdpjMm8ZC4Wle5yfAvgURGpKiJVcC4SPQKIiLQSkSxgKDBdVX8GckXkKPe9pwPTE23AkqcxJuOiOCWK4j5KES/O3T4jgBdxzoN+D7zgvmUYcI+IfA/UBCYl2kaJh+0iEvfNqnpR4h/BGGMSS+ZWpNLeJF/c3T6qOhv404zgqroAOKw08eOd81xdmkDGf34OoazT/TrfYq99b7xvscMsaCNmMimMY9tLTJ6qOq7w3yJSDWgFfINzEvaPMmibMaaCiLj/JVonSBKeRhCRzsCPOOcPmgC/isiRfjfMGFNxJDzfGQlc2fakzsHeCfQCVqvqEpwrUff52ipjTMXi/dV23yWTPKur6reFT9yrWHZ/qDHGM+W1DEeeiNTBvQPfy1Ia6QhrOVm/S756Gb/+njVY9OKVtNm7Hh3aNObHl69ixv0jmXH/SE7pcQAAd17cnzmPnc+M+0dy6P6pz79aFqVwP5s7lz49u3saM8yfFfBnn6QiGokk9QiSZHqQ44H3gcYi8jTQBzjH11YlIbZ06txPP+Waqy7n+ZderdCxvYxfKSvKA1edyJZteQB0aNOESc9+xH3PfLRznX5HCq33rkfXUQ+TU7sar941nC5nx51Lwfd2l+SuO2/n6SlPUb1GDc9iQrg/K37tk1REI4nLbAQteSbsearqG8Ag4AbgI6CLqr7od8MSCWs5Wb9LvnoVf+KYY3n0lc9YtsopEnawNOHYI4SZD5zNQ9cMpGa1KrRtXp9Zc3+goKCA1ev/ID8/n4Y5NTPa7pK0aNGSZ55/ydOYEO7Pil/7JBXl9bAdoDKQBeS5j6SJyJ0i8kQp25VQWMvJ+l3y1Yv4p/U7mJXr/mDWZ7sOE+d99xt/+/tb9B7zGIuXruHas3qwYNFyenduTaWsKM2b1KFt8wZUz66SsXbHM3DQyVSuXNmzeIXC/Fnxa5+kIhJJfOgeuuQpImcC7wKH4sxm8qGInJxMcBHpCQxPq4UlCGs5Wb9LvnoRf/iAjvTs1JIZ94+kfatGPH7dKbz96X/5Up25El774FsOat2Y2f/5gTkLfuKtSWdx8ZCj+FKXsmZDarcAB60UbrLC/FkJkkiSjyBJpud5GXCwqg5X1dOAzsCNid4kIjnABOCW9JpYvLCWk/W75KsX8XuPeYw+Fz5O3wsf56sfljNy/As8P3EYndo2BeCYji35UpfSaq+6rFy7mV6jH+Ouf39IfkEB6zflZqzdmRDmz0qQlNcaRttUdedEoar6i4gkc+j+CHAtzjx6ngttOVmfS776Ff+iO1/jnkuPZ9v2HaxYvZHRt7/Ktu076NO5NSOO60jutu1ccvfrgWu338L8WQmSZG6CD9pN8iWWHhaRQ9x/jgH+wEmGO3BmJcmONzGIiJwN7K+ql4nICKC7qo5IpkFWejgzbGy7ScTP0sPtL3yAqns2iLvu1nW/89X9Yzzdfjri9TyLXlEfEPPvAiDerEpDcG5tmg/kADVF5B5VvTS1ZhpjyrPyNjHIvqkGVdXehf+O6Xla4jTGFMuPKen8lvCcp4jUwxnPXhPnglcW0EpVh/ncNmNMBRGNJC7DEbSb5JO5YPQcsAVoB8wEeuNUr0uKqj6BU8rTGGOKlcytSMFKncndqrSPqg4ApgEPAEcB+/naKmNMhRLGse3JJM/l7v8XAQeo6m84I46MMcYTYRyemcxh++8iciXwCTBORDYA1f1tljGmIgnjBaNkep7nAltVdQ4wD7gJuNrXVhljKpZkep3Byp2Je56q+jtuGU5VvRpLnMYYjyVzTjNo5zzjlR7eiDsBcnFUtbYvLTLGVDjl6iZ54IAya4XJOD+HULa59DXfYuvdx/sWO2jn2MqzCIn3d9B+G/FGGP1clg0xxlRcURJfgEl28uGyUj4nBzTGhEo0msQIo4BNq2TJ0xiTcWGcki6p5Cki1YBWwNdANVVNbbpwY4wpRrm8z1NEDgd+BN4EmgK/isiRfjcskbCWfA1T6WE/YkcjcMfQDrx0aReev/go9qlXnbo1q/DYqEN5/uKjeOnSLuxTzxmDcX6vVky/uhvPX3wUPds1TLndeXl5jBxxBr2OOZquR3bmjde9u4AV1vLAZdHu0ijseSZ6BEkyPc87gF7Av1V1iYicDtyHU9MoY8Ja8jUspYf9it3rwEYADLpnDoe3qsv1A9uxfkser8z7jTe+XMoRrevSsmFNsitncWKnppx4pzMHzUuXdeGj/64iN29Hqdv99NQp5NTN4fEn/sXq1as54rBDOO74E0odpzhhLQ/sd7tLK4y3KiVzAau6qn5b+ERVpxGAc6VhLfkaltLDfsV++6vlXPPMAgCa5VRn1catdNo3h0Z7ZjN1zBGc1KkZnyxaTetGtfh00Wq2bs9n6/Z8flq5mbZNU7u1eNDJg7lh7M07n3tZRC2s5YH9bndpRZKYFCR0h+1AnojUwb1hXkQk2eAi8q6IfCMi891H51QbWlRYS76GofSw37F35Bdw92kHM+6UA3hz/jKa1a3O+j/yGPrAJyxdu4Xze7Xi+6UbOKxlXWpUzWLP6pXpuG8O1atkpdTumjVrUqtWLTZu3Miwvw7mxphEmq6wlgf2u92lFU3yESTJfAWPB94HGonI00Af4JxEbxKRCNAGZ0o7z38rYS35GobSw2UR+7IpX1K/VlVevaIrG/7IY+ZCZ/KuWV8v58rj2nL3NOXJDxbzr/MP56dVm5n/01rWbN6WctuX/PorQwYP4pzzzmfIqUNTjlNUWMsDB63dzk3yidcJkoTJXFXfAAbhlBv+COiiqkXrGxWnsIf6togsEJExqTfzz8Ja8jUMpYf9jD3o0GaM7t0KgC15O8gvKGDuD6vp4V4Q6tyyLv9dtpGcmlXIqVmFk+/9iLEvfE3jOtXQpRtSaveKFSs4vn9fxt8ykeEjzkopRknCWh44aO0O43yeyZThyAHWAM/GLlPVNQneWgeYDVyIM//neyKiqjozjfbuFNaSr2EtPexV7OkLlnHXsA48f/FRVMqKMO7Fb/h2yXpuH9qB07o0Z2NuHhc98QXrt+Sxd93qvH5FV/J2FDDhlW/IL3GmhfjuuO0W1q5by8RbxjPxFmcY6iuvT6NatWqpBYwR1vLAQWt3NApZCbpy0YAdt5dYeriQiOTz5wlClqlqqeoCi8ilwN6JCsFZ6eHyx8a2lw9+lh4+cdxkataNfzvaptUrePXGszzdfjqSmZJuZ74XkSrAUHYdkpdIRLoAVVV1trsoAuSl2E5jTDlWXm9V2klVt7kF3XonWhfYE7hDRLJFpBYwHHi59E00xpR35fImefecZ6EI0AnnfGZcqvqGe2vSlzjlih9U1U9SbagxpvyKuP8lWidIkrk3YRXOOc/Clv8OXJRMcFW9Hrg+taYZYyqKSBI9y9IetovIjcBf3KdvqupVItILuBuoBjyrqte563YAHgNqAx8A5yW6xTKZ5Hmoqn5eumYbY0zyoiQxq1Ip4rlJsg9wME7n7y0RORW4DegG/Aq8KSL9VHU6MAU4W1U/FZHHgVHAQ+m2Z0op2myMMaVWOKtSokcpLAMud6/T5AHf4QzaWaSqi91e5RRgsIjsgzNb3Kfue58ABifaQDI9z69EZCgwB9hUuDCJ+zyNMSYppZzPs1kxo8TXqeq6wieq+k3hv0WkNc7h+/04SbXQMqAZ0KSE5XElkzxP5M9ZuADnIpAxxqTNuUk+0UzyO//5YTEvjwPGFl0oIu1wptO8EtiO0/ssFAHycY7AC4pZHle86plVVXWrqmYnCmKMMekoZc+zK7CkyMvrijxHRI4CXgQuUdVnRKQb0DhmlUbAUjdWccvjitfz/AQ4JFEAY4xJVylvkl+SaISRiOwFvAIMUdV33MVznZekFbAYZ8DPZFX9WURyReQoVf0IOB2YnqjN8ZJnsG6qMqH133u8mXi4OPWHPelb7JX/Hu5bbIBEQ6PTEbahpVEiRBOknESvF3EFkA3cHXN+9GFgBE5vNBuYBrzgvjYMeFREagNfAJMSbSBe8swWkYMpIYmq6heJ22+MMYl5PTxTVS8GLi7h5YOKWX8BcFjyW4ifPFvgZOjimlzgvm6MMWmLkMRN8mXSkuTFS57fqurBZdYSY0yFlcx8naGbz9MYY/wWxlmV4iXPD8qsFcaYCi2MPc8Sh2e6J1wDK+j1yTMR2+/4YYldr3Y23z14Cm2a7Kq2eesZh3JCL1J7AAAYD0lEQVRWrza7r1erKl/eO5CqlVOfotzPfRL2evOlUdjzTPQIkoBNbJ+82LrTN0+YyDVXXV7hY/sdPwyxK2VFmDTqCHK3ORPi1KtVlRev6Un/jnvttl7Pg5rwyrW9abBHemNA/NwnhfXmZ737Aa+8Po3LLrnQs9h+fw5LKxqJkJXgEZqeZ9AFvT55JmL7HT8MsSec1onHZynL1m4BoEZ2ZW59YQHPfPjjbuvl5xdwwviZrN2UekVOL9tdnDDXmy+tSJKPIPE1eYrI8SIyT0S+E5H7vIwdhvrkZR3b7/hBjz2sW0tWbdzK7AW7Rtb9vHIT835Y9ad13124jDWbtqbeYJef+yTM9eZLK4zVM31LniLSAueO/pOA9sAhItLPq/hhqU9elrH9jh/02Kcf05oeBzZm2g19ObB5Do+M7pr2YXkifv8+l/z6K8f27sGpw04r1/Xmree5u4E4MzUvcefTG4IzttQTQa9PnonYfscPeuxjx75Fv3Ez6H/TDBb+tIZzH/yQ39fnetbG4vi5TypSvfkwXjDy86umFbBNRF4D9gbewMOSHEGvT56J2H7HD2tsP/nZ7opVbz6ZyY6DlT0T1m1PlYg8ChwJdMeZRPk14N9u9c1472uO1W03SbKJQYrnx8QgftZtv3TSVOo0aBR33bW/L+eei4Z6uv10+HnYvhyYpaorVXULTtnhUg28N8ZUDD6U4fCdn4ftbwBPisiewEagH878esYYs5tkLggFK3X62PNU1bnA7Ti1j74FfgYyfWLFGBNAWRES3iSfFbDs6eu9Cao6GZjs5zaMMeVAMoflFeiw3RhjkhLGw3ZLnsaYjCtvU9IZY0yZ8KGGke8seRpjMs56nsYYk4KI+1+idYLEkqcJNT9HAdU59jbfYgOsfetqX+OHifU8jTEmBZEkznlaz9MYY4qwnqcxxqQgShIF4KznaYwxu4tGnEeidYLEkqcxJuPCeLU9tAXgwlIGtySfzZ1Ln57dPY8b1v0Sltj196zOoqnn02avnJ3LhvRoy3uTTtv5/K7RPfno78OZcdepzLjrVGrXqJLxdpdl7JQkM4t8sHJneHuesaVT5376KddcdTnPv/Rq4GMD3HXn7Tw95Smq16jhWcxCYd0vYYhdKSvKA5f0Zcu2XYXS2rdswPBj2+82qUWH1o044ZrnWL1hSyDaXdaxU2E9zzIUhjK4JWnRoiXPPP+SpzELhXW/hCH2xHOP4dE35rNs9SYAcmpnc/PZ3bjyodk714lEoGXTOjx4aV/euXcYZxybem2gMOwTrxSe80z0CBLfep4icjYwJmbRvsBTqjqmhLeUSkmlU72oAOhnbICBg07m559+8iRWUWHdL0GPfVqfA1i5/g9mzVvMlaceTlY0ysOX9+Oqh95hy9a8nevVyK7CQ698zqQX/kNWNMpbd/6VL3Q5Xy9emZF2ZyJ2Kpyj8kQ9z2DxbU+p6mPAYwAi0g5nFvmxXsUPehncTAnrfgl67OHHtqegoIAehzSnfcsGzHv0LH5avo5JF/chu0ol9tu7Lnec35OrH3mHB1/6nC1bnUP79+f/woEtG6SUPIO+T7wUxvs8y+qw/SHgb6q6yquAQS+Dmylh3S9Bj937sqn0ufxp+l7+NF/9+DsHj3yMdmf8g76XP83p41/l+19Wc+VDs2ndLIfZ9w4jGo1QKSvKkQc0Y/6iFRlrdyZipyKMddt9/6oRkV5ANVV93su4Vga3eGHdL2GNXZT+sppnZ3/DB/efTt72fP4982u++zm1PkN52SfJiLqlNhKtEyS+lR4uJCLPAy+p6tNJrt8cKz1sAsAmBtmdn6WH7/zny9Rv2CTuuitXLOWKMwd6uv10+HrYLiJVgG44NduNMaZYkST/CxK/D9vbA/9V1c0+b8cYE2JhvGDkd/JsASzxeRvGmJCzAnBFqOpzwHN+bsMYU04ELTsmEI6bF40x5ZpfwzNFpDbwMXCcqv7k3v1zN1ANeFZVr3PX64BzX3pt4APgPFXdXkJYIMTDM40x5UeiSUGSOSdalIh0BuYAbdzn1YDJwIlAW+BQEennrj4FGKOqbXD6wKMSxbfkaYzJOJ9ukh8FjAaWus8PAxap6mK3VzkFGCwi++Dci/6pu94TwOBEwe2w3RiTeZHIbjNTlbSOq5mIFH11naqui12gqmcDxKzbBFgWs8oyoFmc5XFZ8jTGZFwpb1X6sJiXx5F47owoEDsqKALkx1kelyVPY0zGlfJWpa78+RbIdSS2BGgc87wRziF9ScvjsuRpTAn8Hj657+gXfYu9+MGTfYvti9JlzyUpDs+cC4iItAIWA0OByar6s4jkishRqvoRcDowPVEwu2BkjMm4shieqaq5wAjgReBb4HvgBfflYcA9IvI9UBOYlCie9TyNMRnn5/BMVW0e8+/ZwEHFrLMA52p80ix5GmMyzoZnGmNMKkKYPUN7zjOsZVn9Lvnqd3wrmexP7GgE7j6jI69e2Y2XrziaferVoF2zPXjj6u68emU37j6j426HrXVrVuGjm/pStVJqf8JBKz0cxinpQps8Y0un3jxhItdcdXmFj+13/LvuvJ0Lzj2b3Nxcz2IWCus+9yp2n/bORMAn3vE+t7/2LWMHt+fy49py95vfc+Id71O1UpReBzp303TfvyFPX9yV+rWrZrzdXvFjeKbfQps8w1qW1e+Sr37Gt5LJ/sV+a8FSrpzyBQDNcqqzauNWFv66jjo1KgNQI7sS23c4923nFxQw5N4PWbd5W8bb7aUwJU7w+ZyniJwG/J/7dLqqXuFV7LCWZfW75Kuf8a1ksr+xd+QXcN+ITvTr0IRRj3xKnZpVuOXUg7mkf1s2bMnjY3UqcH7w3e+BarcX/JpVyU9+1m2vjnOvVBucu/8/EpFeqjrLi/hhLcvqd8nXoJWUTVZY97nXsS9+Yh7ja1dl2jXHUK1KJU664z3+u2wjI7q34MbB7fnb0/O9aHbgPidhnEnez8P2LDd+DaCy+9jiVfCwlmX1u+Rr0ErKJius+9yr2Kd03psLj3UmsNiybQf5BbB28zY25TpTSq5Yl8se1St702iC9zmx0sMxVHWjiFyPcxf/H8D7OJOSeiKsZVn9LvkatJKyyQrrPvcq9ptf/sa9wzvx8hVHUykryg3PLWDtpm08dPZh7MgvYNv2fK5wz4kGqd2eCeGtSr6VHhaR9sCTQF9gPc7ceZ+p6h0J3tccKz1sKoCwjW33s/Tw48++ScPGTeOuu2LZb4wcMsDT7afDz8P2vsBsVf1dVbfiTDDa3cftGWNCKoy3Kvl5hngBcLuI1MA5bD8e+I+P2zPGhFQIj9r963mq6tvA08DnwFc4F4wm+rU9Y0yIhfCKkd+lh28DbvNzG8aY8ItEIkQTHJcnLNNRxoJ/A6AxptwL42G7JU9jTOaFMHta8jTGZJwNzzTGmBSEcXimJU9jTMaF8KjdkqcxJvMiJNHzLJOWJM+SpwGc6dD8khUN2sc+GPwsD9xoxBTPY0a3rqO+51ELha/vacnTGJNxds7TGGNSEL5+pyVPY0wA2AgjY4xJRQi7nqEtAFfIj1K4YShVm6n4d95+Kz2OPpIuh3fiyX8+7lncsO7zMMWuV7sqX983kNaNa7Nvw5pMv74P067vw10jDtt5PvGmUw/h7Rv78s5N/TijeysPforkhHBekHAnT79K4YahVG0m4n/w/nvM/eQTZr03h7dmvceSJb96Fjus+zwssStlRbj3rM7kbtsBwC3DOjHhhQX0v/ltIhEY0HEvurZtSIuGNekzbgbH3jSDS45vxx7Vq3j148QVxvk8Q508/SqFG4ZStZmIP3vmDNodcACnDh7E4EEn0K//cZ7FDus+D0vs8UM7Mnn2Ipavc8qIHdQ8hznfrQBg1oKldGvXiM9+WMnof3wKQEEBZEUiO8sd+y2S5H9B4mvyFJFrRERF5CsRudbr+AMHnUzlyt4VxSpUUlnWoMf2O/7q1av44vPPeerp57jv/ocYOfw0vCrjEtZ9HobYQ7u2YNWGXN5ZuGznsthe3KbcPGpXr8LWvHzW/7GNSlkRHjrvSJ54dxGbt3r32YwrhMftfpYe7gUMBQ4FNgMvi8ggVfW+q+ixMJWqLcv4OTl1adNmP6pUqUIbEapmZ7Ny5UoaNGiQduyw7vMwxD6tW0sKCqD7AY05cO86PHzekdSvnb3z9ZrZlVm/eRsAe1Svwr8u7sqc737nnte/Sf+HSFIIrxf52vM8GJihqhtUdQfwFnCSj9vzTBhK1WYi/hFHdmHm2zMoKChg2dKl/LF5M3Xr1vUodjj3eRhi9x8/kwETZnLchJks/GUt5z38MbMWLKVL24YA9DqoCZ/o72RXzuK1v/Viyvs/cscrCz37OZIRxnOeft6q9AVwj4jcilPD6ARCco41DKVqMxG/34Dj+GjOB3Q7qjP5+fncfd8DZGVleRI7rPs8rLGvnfo5k0YeTuVKUf67dD2vfvYL5/UVmtevyfBjWjP8mNYAjP7Hx/y8crNn2y1JGKek8630MICIXAaMANYAs4DDVTXuVQYrPZwZNra9fPFtbPtX94MPpYdfenMmTZrELz28dOlvDBrQ29Ptp8O3nqCI1AJeVNX2qtod2Ar86Nf2jDHhVTirUtxHphtZhJ+H7fsC/xKRTkANYKT7MMaY3YTxsN3P0sNfAS/ilB3+DLhXVT/ya3vGmPCyC0ZFqOrNwM1+bsMYE35hvFXJJgYxxmReCLOnJU9jTMY5uTPROc9gseRpjMk4m0neGGNS4MdRu4gMBa4DKuNcsH4wlbaVJBQjfowx5VwkQiTBozRdTxFpCkwAugAdgHNEZH8vmxzEnmcWwIrlyzPdjgol38cRRlEbYVTmolvXeR9z24bCf3ozJjfG7yuWJ8yNv6/YmROaiUjRl9epauwP3Qt4R1XXAIjIC8ApwE1etBeCmTwbA5x5xrBMt8OY0PKvRDDg/I16NVpwA7D2zDOG1Uly/Vzgw2KWjwPGxjxvAiyLeb4MOCyVBpYkiMnzP0BXnB92R4bbYozZJQsncf7Hq4CqukZEWgG10wxVtKsdBWIPpyKApzM7By55qupWYE6m22GMKZbn81O4h9ZrPA67BKcTVqgRsNTLDQQueRpjjAdmAWNFpD7OZOwnA+d4uQG72m6MKXdU9TfgWuBdYD4wVVU/83Ibvs7naYwx5ZX1PI0xJgWWPI0xJgWWPI0xJgWWPI0xJgWWPE0giEiNTLchFSLSNqxtN+kJ5X2eItICZ0TCN0C+Wxc+8ESkPc7NuouBX9wBAV7Fbg3UAb5W1T+8iuvGPgjYT1Wf9TJuTPwTgGNE5DpV9bTOrbvPmwE/AD+oqmejTESkF/AgMMCN7xm33Y2B74FfPW53W6Auzii+ZV5/XiqK0PU8RWQQ8CZwB/AEcImI7OFh/INFpJFX8WLinghMAc7HmZzgeA9jnwS8ANyKUzfKq7gREakEnA38n4gMjH3No20chzMmeaoPifN4YCpwFXAcHk5oISJ9gDuBmsDhXsV1Y5+E81kZDTyCO9+Dh7H/jfM5vB64Q0SSHVduYoQqebqHR6cBQ1W1N/AqsA/wNxFJa2xsTDK4HHhORBqm1djdYzcCLgH+oqoDgW+BoR7HHqKqPYF6InKCiDQXkWrpxFbVAlXdjnOT8Vqgv4iMKHwtzaYX9q7uBY4BGorIEyLyLxE5Pd0vRBGpAvwFOMstff0M0FZE9kp3v7iJ816cL5XLgAbu8rS/UNzP+AhguKqeAGwHuotIS7ecdzqxGwAXAyNUdRjOF20P4D4RyUmv5RVPqJInzkD/+sDe7vMXgNeBbOAMEUm5ZxGTDPKBA4CnPEygeUAV9wHwKFA33T+GmNg1gI3uH0dLnF7F/Ti9xWwPtrEaWA98DHQVketF5ApIO2FswPkM3ojzR/0w8D/gYOCCdH6fOBNBtATquF+szwG3AZNwvmyrpxLUTcqHA6NUdR6wBThTRBp78YWC8xlvABzu/j4PBP6K08v9e5pfKnk4fyu1AVT1dZxhjOuBi9Lc3xVOqJKne27mcaCfiBzkngd6B5gHHM2u5JSOn4EjcM4HPe1RAl2Pc/i1TUSiOF8ADXBnfXF7Fam2fT1wtTscrS1wgar2c7fXxt1OuuYCC1T1n8BK4Aqc86tp9UBV9SfgDJxzhi+o6qfAeJwpxw4hjd+nez55Kk7PajzwiLtfHsfZL81SjLsNuFVVP3KTzbs4Xyp7AaSbgNzP+DjgQpzTU4+r6vHADTh/r+3SCL8eeBv4q4icIiITgT2B93D2txdftBVGqJKn602c6af+6ibQHar6FJADtPIg/jPAT6o6HPgNDxKoe+j7HLDITfiVgR2quklEhuP8cafUE1LV7ar6jvvv91X1Gfffb7gxvUie+TgT0A4GjgUmA61EZIgHsT/GKU/9Dez8eV7G+X3uk2bsV3EujByC03su3C81gKapBlXVPPf/O1R1I05SurVwmfsFmTJVnQF0Bj7HOWWCqi50X075/KT72XsE54LlyUAtVT1dVV/E6Y16fq6/PAvl2HYR2QsYA9TD+SYtwJkEoLeq/u7xtv4JtAf6eRXbvdp5FfAaTrtHqOrXHsStDOwHVMNJmjcCx6tqWtPyu72px3H+oEer6jsicgYw2+3xpiUm2RwNbMPpmd8E9FTVVWnGbgXcB/zCrlM8Y4EBqvprmrEjhT1vEXke+E1VL0knZpH45+McTbyE00O8DjhZVX/2IHYU50JULXcbN+Ls79Xpxq4owtjzxP3Q3wHMBk7HuXI93MvEWfgHrapn4kz+mtZFhiK2A8Nxaqyc7kXidGUBRwF3AxcAZ6ebOMHpTRXGLOzl4lwdTztxuvHzcXqDHXF+r+fh/D7TSpxu7B9w9sX/cA6FB+Ps87QSpxu7ICbxPwBU9fjCy6s481xeB4wCRnqROF0FwEHAv4CROF/gljhLIZQ9z1juucKCwkMpj2NHvby/LiZudZxD3xtU9b8ex64EVAWi7iGlp2J7Wz7EzsL5koqq6oZE66cQvwoQ8fL+2pjY2UAlVd3kcdwIu/aJp7Hd+DVw9onnscu70CfPsBKRKu7FB2NMCFnyNMaYFITynKcxxmSaJU9jjEmBJU9jjEmBJU9jjElBKKekM7sTkeY49bQXxiyOAPep6uQ0Y7+BM3TyCRGZD3RX1XUlrLsH8LKq9ijlNk4BxrgTeMQu7w48oKoHJHh/AVC/NPeFisgTONP33VmathpTyJJn+bFFVTsUPhGRpsDXIjJPVb/yYgOx8UtQBzjMi20ZE3SWPMspVf1NRBYBbUTkEJxRJDWA9ap6jIiMxBl5E8UZ9z1GVb8XkSbAk0ATnElSdo6Nj+3hicj/4YyS2g4swplG7Z9ANbeH2hFnAo77cMaXZwGTCnvCInITMMzd9qJEP4+ItMGZeLgWzrDC+TjT8OW6q0wQkUPdn+c6dww7Jf2cpdqZxhTDznmWUyJyBM5EKXPdRe1wDrmPEZFuOImvq6oeDNwOvOyu9yDwqaq2Ay7CGStfNPYJOMnyCPeQejHOXANnsqsHHMGZMvAaVe0IdAOuEJHD3YmhTwY6AEcCyUyzNgp4UlUPd3+ufXFmYyr0P1U9BGe+1ydFpH6Cn9OYtFjPs/wo7PGB83tdBQxT1V9FBOCrmCGPA3AS0Mfua+DMe5kD9MKZcg5V/UFECseyx+oFPK+qa931LoOd514LtcGZT3NyzDaq4czVuT/wUuHwURGZjJOo47ka6C0iV7mxm+DM4l7oYbctX4vItzjTCnaJ83MakxZLnuXHlgTnJGPHLmcBT6nq1bBzEpQmOLPFF+D0GgttLybWdnc93PfviTPrT6wsnFMEsedhG+JM33ZHEtso6mmcz+tzONMS7l0kRmwdqyjOxL/xfk5j0mKH7RXTDOBUESmsjXMezgxVAG8B5wCIyN44JTKKmgUMkl2lT8bilKPYDmS5k1kosEVETnNj7QV8jXMudDowWET2dBPa6Um0uS9wk+4qQteZ3WsSjXC3cwi7TlfE+zmNSYv1PCsgVX1bRG4DZopIPk45jEHuFGujgX+KyHfAEtzJeIu8f5qI7A985B4Of4NzTvIP4DP3eVfgRJz6OFfhTAB9vap+BCAiB+JUAFgLLMCZwzOevwEvi8hmnN7r++w++XULEfkSp0f8V1VdA8T7OUuzy4z5E5sYxBhjUmCH7cYYkwJLnsYYkwJLnsYYkwJLnsYYkwJLnsYYkwJLnsYYkwJLnsYYk4L/B0XR+PBAD9iyAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_confusion_matrix(confusion_mtx, classes=range(10))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 进一步探究预测错误的图片"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "目的：解释误分类的情况 —— 误分类的图片有哪些，这些图片对应的预测类别和正确类别分别是什么？"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "errors = (Y_pred_class - Y_true !=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "Y_pred_classes_errors = Y_pred_class[errors]\n",
    "Y_pred_errors = Y_pred[errors]\n",
    "Y_true_errors = Y_true[errors]\n",
    "X_val_errors = X_val[errors]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def display_errors(errors_index,img_errors,pred_errors, obs_errors):\n",
    "    \"\"\" This function shows 6 images with their predicted and real labels\"\"\"\n",
    "    n = 0\n",
    "    nrows = 2\n",
    "    ncols = 3\n",
    "    fig, ax = plt.subplots(nrows,ncols,sharex=True,sharey=True)\n",
    "    for row in range(nrows):\n",
    "        for col in range(ncols):\n",
    "            error = errors_index[n]\n",
    "            ax[row,col].imshow((img_errors[error]).reshape((28,28)))\n",
    "            ax[row,col].set_title(\"Predicted label :{}\\nTrue label :{}\".format(pred_errors[error],obs_errors[error]))\n",
    "            n += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXgAAAEUCAYAAAAhqy2HAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAIABJREFUeJzt3XucnOP9//HX7MxudnMQOck5gnCpY9SplDqfKepUlF+qRYtSqkppKaUHqs51aIPSklbQOldDKfKNM3H6CJKS7JJEBDnvYX5/3PfeM9fI7M7Ozs7O3vt+Ph55PK5rrvtwzXwm115z3fd9XYl0Oo2IiMRPVXdXQEREuoYaeBGRmFIDLyISU2rgRURiSg28iEhMqYEXEYmpVHdXwDk3HngXmJn1cgK40swmd/LY9wN3mdktzrmXgZ3NbHGebQcC95jZrh08x6HAKWa2c87rOwPXmNkm7eyfBoaZ2cIOnPMW4DUzu6yd7Q4GfgG0AIuA483s3ULP0xmKa9fFNWv7g4DbzGxAoefoLMW1Z/1/7fYGPrTczCa2Zpxzo4HXnHPPm9mrpThB9vHzGARsU4pzVQLnXB1wO7C5mb3jnDsduArYr4zVUFy7iHNufeAygsa13BTXEuuq/6+V0sB7zGyec24WsIFz7svAd4B+wKdmtotz7jvASQRDTB8T/EV+yzk3CrgVGAX8D1ir9ZjZf3mdc+cA/w9oAmYBk4Cbgbqw57AlsAFwJTAESAJXtfZQnHMXAkeH557V3vtxzm0AXAsMAEYCLwNHmNmKcJOLnXNbh+/nPDO7P9xvte+zjfNsBfwx/M+RJPjPPzAs7g+syLdvOSiuJYkrzrm+BI3BGcBf26tnV1NcK/f/a0WOwTvntgMmADPClzYm+Lm2i3NuJ4Jg72hmWwC/Be4Jt7sW+D8z2xg4FdhwNcf+OsEXZLvw59hs4BTg22R6JgngLuBsM9sS2Ak40zn3FefcgcAhwERgezIBacvxwK1m9pXwfa2D/5f5PTP7MvAt4Fbn3LB23udqmdnzrY2AmS0Bvgc845yrD9/jTwqoa5dRXDsf19AN4b+S9JY7S3Gt3P+vldKDb/1LDEGdFgJHm9kHzjmAV83ss7B8P4IP/ZmwDGCQc24wsDtwJkD4M+ex1Zxrd+DvZvZJuN0ZEI0tttoAWA+YnHWOOmALYCPgbjP7PNxvMsGXsy0/AfZwzp0VHnsUwV/oVteHdXnNOfcGsB2wQxvvs13OuU2BnwMbmdm7zrlTganOuYlmVq75KRRXSh7Xk4AmM5uc897KSXGlZ/x/rZQG3hvTW40lWekkwYWlnwA456oIAvAJkMYfk2xazbGawu0I918TWDNnmyTBz8vsccbhwKfApQWcI9cdBJ/134AHgHE5x2jOSlcBjbT9PguxF/B01kWaa4HfE/yELfgCUScprhmliuskoG/YwNaQaWz3NbP6Ao/RWYprRkX/f63IIZp2PAIc6ZwbGea/B0wL0w8DJwA458YBu6xm/38D33DOrRHmLyAYy2wCks65BGDAcufct8JjjQVeIxjrewg4zDm3ZhjEYwqo817AhWY2JcxvS/CFaDUpPM+XyfzUbet9FuJFYKfwiw5wEDC7I1f/y0xxLYCZbWNmm4SN2b6EjW0ZG/eOUlwL0yX/XyulB18wM/uXc+43wKPOuRbgM+AbZpZ2zp0M3OycexOYS3BxJHf/B51zGwFPhz+lXicYc1sGPBvmdwQOBK4Mf6ZVAz8zs6ch+jn1PMFf51eAYe1U+6fAPc65pQS9iicIvhit1nXOvUTQU/mmmS0C2nqfqz1J9kUbM3vMOXcp8B/n3CqC264ObKee3UZxLSyu7dSn4iiu3fv/NaHpgkVE4qknDtGIiEgB1MCLiMSUGngRkZjqcRdZAZxzVwFfC7MbETz8sDzMb2dmy1e7Y/HnG08wl0T/drabAxxqZs934NgXAEPN7JR2tvsBcC7wYfjS52a2Y6Hn6Ql6aVwPB84nuCtkLnCSmf2v0PP0BL00ruOA64DRBO3smWb2SKHnKZUe2cCbWfSgQhikozsSpB5qe+AMM+v2R9O7Sm+Lqwvmk7kB+JqZzXTOfY3gicytu7dmpdXb4hq6D7jezP7gnNsCeMw5N8LMVpazEj2ygW+Pc24l8A9gc4I5KJ4jawY4589zcQBwHsFDI8sI/tJOb+PYwwn+Uw4HRhDMoXG4mc0PNznZObc50Af4nWXmw+joeUYBD5J5gGV7YA3n3NlAQ7j/zHz7x1Hc4hq+j1da42hmTzrnxjvnxpvZnA5/QD1UDOO6FjDYzP4AYGYvOed2IJglsqziOgZfA9xnZq6tnkLYg7qEoBHdguChi7udc/3aOPY3gelmth2wLkHwsx+eWG7BPBV7AL9yzm1czHnMrL71AZZwu7eA35jZZsCfgIecc23+BI2hWMUVeAnYxDnXOonYAQRPLo7Mt39MxS2uGwBznHOXO+dmOOeeBkaaWWP7H0VpxbWBB/hvAdvsQfCfaZoLHvf+C8Ff2Qn5djCzKwnmmziDYIxtE/x5Km4It6sH/gXsVsx5cs651Mz2MrMnw/zfCB7aiNVP+QLFKa7vAscB14cPzmxJ8CDOqkL2j5nYxJXgQauvAk+Y2bbA6cCUsJdfVrEcogktycknAJxzNVmvJYFpZnZE6wsueMw572Pf4dNq2wCTgccJgtnePBWpNs5zcHtvxDm3NvB1M7s65/2UvUdQAeIU1z7AOxbMWtia/yHBRcjeJjZxDbdbbGb/ADCzZ51z7xEMQZV1Sok49+CzLQC2CtNHZb0+DdjTObchgHNuX4IpWOvaONZewBVmdhswn+Cv/ermqRhHMBPetCLPk20p8Evn3DZZ+/cleFS7N+vpce1D8Aj+2DD/Q+Cp8NH33qynx/UZYIVzbv9w/w0JZrss+/TOce7BZzsVuNY5txh4lOAiJWb2hnPuBOBOF0xa1ETQU87tTWS7ELjMOXcRwV/7p/B/utU6514kGFf8gZm9DZDvPC7/PBXeRdbwdrobwh7NZ8DBZtYbf8pni0Ncjye4npIE3iRscHq5OMR1L+Bq59yvw02OM7N5Hf8oOkdz0YiIxFRvGaIREel11MCLiMSUGngRkZjq1EVW59xRBE97VRNcqb62gH36ENy/3YB/i5KUV5LgXt/nSvH4tOJaUUoWW8W1onQ4rkU38M650cDFBA9nrCR4mOBxM3ujnV23prCHGqQ8diS4s6CzFNfKU4rYKq6Vp+C4dqYHvzvwWOs9u865u4BDCW5LaksDwNx5S2lq1h083SWVTDBmdD8I41ECimuFKHFsFdcKUUxcO9PAj8o5UQPBE2PtaQZoak7T1KQvTAUo1c9uxbXylCK2imvlKTiunbnIWkWw6GyrBN0wW5qIiKxeZxr4ufiz3o2gzPMsiIhIfp0Zovk3cIFzbhjBXCmHEEyrKSIiFaDoHnw4r8K5BDO0vQz81cx6++RXIiIVo1P3wYfLx8V2CTkRkZ5MT7KKiMSUGngRkZhSAy8iElNq4EVEYkoNvIhITKmBFxGJqd6yJquISFHuHrxTlN73tV/m3a7prae9fP9dz+6yOhVKPXgRkZhSAy8iElNq4EVEYkpj8F1kwpqjovS/x6/hlY148KYofc2Xz/fKzvzwsa6tmIi0afEpW3n5Pmf9PEqnm5vy79hSebOlqwcvIhJTauBFRGJKQzRt2GrY+l7+4ubhebfd/vove/nUlntnMjV1Xln2z7yTp5/llX3vje2jdP/98t+SJeXx25G7evmT/piJT2riHl7Z1M0viNJHL3y8S+slnfOlwWO9/LOX7R6lU7sc4W+cKKwfnBg0wsvfMmyXKD1pQfd8H9SDFxGJKTXwIiIxpQZeRCSmesUY/K7DN/XyA6pqovSklf29sj1u2zFKV627hVeW6D+49JVL9fGyyU0z43Y3rPWMV3bifN1C2dV+PmoXL3/Srf4YfHLDr0bpdM5tcQfennmknb01Bl9JNho8zsvPuGJPL5/a5ehOn6NqrXW8/EFHLslkrur04YuiHryISEypgRcRianYDtEsefiCKF012vmFyczbTgwYWqYaFSjrlqy6dLobKxJfddX+sNj7e2d+vvf73eleWaJuQAcO3L/9baRsNhm8dpSefvluXlmxQzIt82d7+c9O/1WU/ny+/73aeo6/bXdQD15EJKbUwIuIxJQaeBGRmIrtGHxiZGaagcSaI9rYsrKkP50fpadVr+jGmsRL/6zpIhr+cLhXVr33cVE699bHjph28D+L3lc6L3vMHfxx99Rux5TkHC2zXvTyI5+cVZLjdhX14EVEYqqgHrxzbg3gGWB/M5vjnNsduByoA6aY2XldWEcRESlCuw28c25b4CZggzBfB0wGdgI+AB5wzu1jZg91ZUXb88HW/q2QVYMqZ1im5cN3vfzco6+I0u8uHOSVbThqYZS+tf7Nrq1YjO01YqKXnzr561E6+2nhXEu+f4KXr3/Fv/Vxg2cuj9Lpzxd6ZRcnP+xwPaV9Ow3fJEo/8dFrebf7U8qf7bVUwzJNT0+N0juc/EBJjlkuhQzRHA+cDNSH+W2AWWY228yagNuBw7qofiIiUqR2e/Bm9l0A56Ie8iigIWuTBmBMyWsmIiKdUsxF1iog+xHLBFB5ixGKiPRyxdwmORcYmZUfQWb4ptsMueN3/gsFrsJSDok1/OkQFi/uG6XP4n2v7LXX/leWOsVBVU6M9x6+eZT+29+P9cqS4/0x+WwLv3F8lF73+Tle2R8H7+DlN8hKNz/t3xb57IK326quFGiLoet5+QcmHxSll/7ev3X4xtczKzNt+oQ/zURHtNS/FaUvOXiKV/ZI47wo/crH3T/9QEcU08DPAJxzbgIwGziK4KKriIhUkA53c81sBTAJmAq8AbwF3FXaaomISGcV3IM3s/FZ6WnA5vm3Lr/mJ6d6+dQex+bZsvwSfQd6+c1e+E2U/r83/uuVNT85LUoPvPiJrq1YD5c9JAMw9bnL82zpPyF82643eGUnztfQSiX5eYt/z0Zys8wTqWvc7M8KebpNj9Idmfkze0gG4Oxv3BGlr65/suDjVLrKGagWEZGSUgMvIhJTauBFRGIqNrNJbna6P1PC69P3jdIVt2pTluRGO3r5qvW+HKU/q/VXiPnqFZmx4pkfz+nSelWq7CkIcm+FbMv8b50fpU+c/1be7bJnnQTYf+/5ebaExJpDvPwBI7eM0vc1vFBw3cT34xZ/hsY9F2RuHa4a5s8YmXTbFXWO5vv8WyGvnhefcfds6sGLiMSUGngRkZiKzRDN7E/9mfzSjauidKLclemERJ9+Ubp60jle2flX/CJKH8qcclWpoky9aZ8o3dbTqblWLst81T+7aE+vrGqjzGyF1PjDYsmJ/rZe2VcO9PJ33p1ZZKbfdhqiKdY7i3MejG9cXvJzLHtmbsmPWYnUgxcRiSk18CIiMaUGXkQkpmIzBp+r5dXMY/5VOx9ZmmNmrcy09PzfeGVnvTws736Xbeuv/NPvqhvybNm2vX6bdYvYd4o6RI+3YnJmSop+V+UfH881btrVecsSVZl+TmcW3ZauMfeYa6N0W3FsS9O913n5zZ/9uFN16inUgxcRiSk18CIiMaUGXkQkpmI7Bn/tWe9E6dOeyrmPNudx9EJd9fW/ROlzGmbllObmM/75sD+N6awffi9K973i+oLPn9zxkCj9zVH+4/Z31s8o+Dg92aaPZq5n/GsHfwWf8RdtFaWrdzum8INmrQzV3kJgKy7OnHPN614s/BxStOcWZa5vjSvyGOklS7z8iqZVebaMF/XgRURiSg28iEhMxXaI5uyGx6L0DxYe5pVVjdqwoGM0Tb/Hyz/Q/FFRdVm0/HMv/8Ppg6P0jR04TnrZ4ijdW4ZkcjUsWRSlN81KAww4PjPb5qi+Dxd1/AdHreHlRz18rZe3u2L7XybWqr91lpd/6S9zvPy6r71ZxtqUj3rwIiIxpQZeRCSm1MCLiMSUBhRzNE27LUpvfOr9Xtn7n+Vf3actfWtqvfw130wXdZz0+68XtV9v8fnKZVHastIdsXTAFm2WP5Tq12a59AxrXevP9XHEkX+P0lNidH1LPXgRkZhSAy8iElO9Yojmk1N/7+WH3JV/NsdU1hOQrx77qle20x2Zn+evfDy7zXPWVWdWBnph7AZeWc3pv2pz33we/nZ8fjpWkjFZi7Kve+dxXll62ade/o7l+Z9Ylp4jue6WXv5PU5JR+sNDl3plT3z0Wlnq1BXUgxcRiamCevDOufOBw8PsA2Z2lnNud+ByoA6YYmbndVEdRUSkCO324MOGfE9gC2AisKVz7khgMnAg8CVga+fcPvmPIiIi5VZID74B+JGZrQJwzr0JbADMMrPZ4Wu3A4cBD3VVRTvjP++O9vKH5NkuV5+fXOrln2jMzCQ4866veGVX1DR7+T99vTFznPP8awCFalnkry5/RbK42zSlbTdXuyidHLuJV9Z48yVe3j6ZW5Y6SXklx0+M0hunBnllT+Ru3IO028CbWXTztXNufYKhmqsJGv5WDcCYktdORESKVvBFVufcxsCjwI+B94Dsp3USgBazFBGpIIVeZP0qMBX4oZnd6ZzbCRiZtckIoH61O1eAG3OGNg6albndMLn+tgUfJ3uoZaucS8q3F1e1L2rODO0snHSRV/TU/Ldyt5YS2Gyr/LOELv7H+2WsiazO8qpEJpPO6Ue2t0JLL9duA++cGwvcCxxhZq1z8M4IitwEYDZwFMFFVxERqRCF9ODPBGqBy52LLkZdD0wi6NXXAg8Cd3VB/UREpEiFXGQ9DTgtT/Hmpa2OiIiUSq+YquDJj/xZGPc9MjNj5IN3+mN4yQlbl6VOkZwxxfkHnxql135JY+7l0O/Hk/KW2eyhua90aV3ki46fn1mdbf9vfeCVrXl71qL1Go//An0iIiIxpQZeRCSmesUQTa7sIZu9j7jFK3t4SibdZcM1q5ZHyY8OOcMrGv+yhmW62jdG+nFNbbRjlG589Fav7Pjmd8pSJynMyCf92TyXzHw8Sic3263o4zZnHWf6yoq947vD1IMXEYkpNfAiIjGlBl5EJKZ65Rh8tqfmv+Hls8fk79n2Jq+s/3U3FnWOJd8/wcsf9GzmY39a0w+U3Z/2XZm37IUf+fGY82n+aQyk+/Xf75dZuV/m3a63Ug9eRCSm1MCLiMRUrx+iyZU9ZDPsvpzC+3Ypb2WkS1z50BAvf8aEX0fp+/tUl7s6Il1GPXgRkZhSAy8iElNq4EVEYkpj8NLrXFD/uJ8/p5sqItLF1IMXEYmpbm/gU6kE663Tn7Gj+3r/Bgzo/I+LkcPrGNA/OM7Y0X2pauPdViVg1Mi6Dp+jX78Uo1ezX11tkrFj+ra7/4R1B1CVveZkAdYaVsuaA9u/26Nf31T0eY4aWUcq1bHzdIbiqrh2lOJa+rhWxBBNOg0fzFsW5ZPJBOPG9GPlymWsWtXSxp6Fyz7+6lQlE9T2SZbkXJUgkYDha9XywdylNDalGTiwmmFDa2n4cHn7O5eI4lp6imtAcS1MRTTwuZqb0zQ2tlBdXUWfmirWGFBNoipBS0ua+oblDBhQzcA1qkmE2y74eCWNjS0kkwmGD6slmUrQ1JQmmcz8BZyw7gDem7OElpY0g9asYUD/FGmgsbGF+fNXsNawWhKJoOfwwbxlVFdXMWxIH6qSCRLA4s9W8fnnTQAMHlTDgP7VUT3bU12dYNiQWqqqEiSTCVauauaj+StIp4PyIYNr6NMnSQL4+JOVLFvWDJD3febTp6aKtYbVev85gt5GmqpEgnTrCbuJ4qq4Kq4Z5YhrRTbwtX2qqK6uYuWKZurqktTUJJnz/hLSaaitTbJG/xTz6peRTkNdXZKRw2t5f+4yhg3tw4qVzSz6cBXVqQRjx/T7wrH79k0yoH81c+uX0tICQwf3YeDAGuYvWMG4Mf2iD3vE8Frmz1/BylUtVCVgzOi+rFrVQipZRb9+Kd6fu5R0OvhZ2Z41BtTw2ZJGliwJvnBjR/elb98US5cG+cbGNAsWLqOmuorRo/ryvw+WUlNTlfd95rNyVUtU/3QaFixcwZjRfWluDr4o8+rb7hV1NcVVcVVcM8oR1+5o4JMAqfCvdSqZIJGAcVnjXy0tsPDjFZAIfv61/rUHGNA/RXV1FWNHZ7ZPJhPU1CToW5di8afLSKUSpIEVK5pJJhPRWFYqBf37pVi2vImqqgRVVcFfeq8+qQTVqQQ11VUMX6s2OkeiKkHfuiTV1VUsX94c1WfpskbWGFDzhfGyZNiTSKUSfPrZKmprkwweVEN1dRWpVBWpVKZeS5c1kkolaEkHPYx+/ZLU9knmfZ9VieAvfVtjdNXVVQwe1Id5DctoakozoH81I0fUUR/+5Etlekul+p2ruFZAXLPfM6WJreLag+OaKPfPO+fcDsB/y3pSacuOZvZUZw+iuFakTsdWca1IBce1O+6ieQ7YEZgArBOml4Xp1f37MfBYVv5YoB7YJsz/DHgvTD8A3BCmvwp8Fu6/TnjuLwPfBmYBm4av3wzcAmwHrAxf2wCYB5we5rcHFgGHhvu/B2wOrAs8CMxYTb2PBN4O058D+4bpPcP8T7LqdX6Y3h9YCExs531OBS5p4zNbBzgqfA9bh/nvA//LKp8QfvbP5YlTRymulRHXUsdWce3BcS37EI2ZrQSivz7OOYC0mc1Z3fbOuYXA8qzyOc65NQgC3ULwpfi6mc1xzn07fP0hYC7wIrAwLAP4wMxecs4NBe4Ij/c6cDzBl3YGwRdgR2A/4ErgOKAaONfMpoZ1Gg7cDXwCvAL0y62/c2480Bie+2zgWmAp8CnwH2BQVr3WBO4B0sDhZvYK8Eob73MJ8MlqzrkV8Eczmxh+ToOB24BVBF/4/XL2eXd1n3kxFNeKiiuUKLaKa8+Oa9mHaEREpDy6/UEnERHpGmrgRURiqiLvg2+Pc+4q4GthdiNgNtB6P9F2ZlbSx/rC8bnXzKx/O9vNAQ41s+c7cOwLgKFmdko72+0FXEwQsxbgbDP7V6Hn6Ql6aVwPA84DEgQX7E40s1mFnqcnUFy7L649soE3s1Nb02GQju5IkHoa59xA4K/A18zsdefcZsCTzrmxZvZ5N1evZHphXEcQ3EWymZnNdc6dAlwD7NW9NSstxbX74tojG/j2OOdWAv8guDXqaILbioaZ2cKwPN2ad84dQPCXtobgyvyZZja9jWMPJwjecGAEwa1Mh5vZ/HCTk51zmwN9gN+Z2eRwv46eZxTBHQL7ElxVP8nMXg+L3yDoGQwluIWrV4hbXM2s3jk33MwanXMpYG3g42I+m55Mce06sWzgCYJyn5kdDtGtXV/gnFuf4P7Unc3sY+fcxsC/nXMTzGxpnmN/E5huZr9xziUI7uU9BvhdWL7czL4cBvwl59wMggZ6tefJ9wbMrJ7g/tpWU7LSFwJvm9nstj6EGIpdXMNGYCvgfqAO2LuwjyJWFNcuEtcGHgp7+m4PYCQwLetL1ULwQMErq9vBzK50zu3onDsDWB/YhOB+3FY3hNvVO+f+BewGNLVxnoKFvYHLgX3C4/ZGsYtrOFwxwjm3N/CAc25dM1vckWPEgOLaBeLcwC/JyScAnHM1Wa8lgWlmdkTrC865sQRPpK2Wc+43BE+rTQYeJ3ioIpG1SXNWugpoJPic853n4ELejHNuEHBXeK6vmFmv+ykfik1cw17jpmb2CICZPeyc+wxYD3ihvf1jRnHtAr3lNskFwFZh+qis16cBezrnNgRwzu0LvErwkyqfvYArzOw2YD5BryJ78p9J4bHGAbuH5yjmPBHnXJJgfG82sGcvbtxz9ei4ArXAna0//Z1zuxA0Lm8WuH9cKa4lEucefLZTgWudc4uBR4EGADN7wzl3AkEwEgQ/zb5uZrm9iWwXApc55y4i+Gv/FP5Pt1rn3IsE44o/MLO3AfKdp43xxuyLrDsBXwH6A89n7XOMmc3s2EcRKz06rmb2nnPuu8DU8ELiYuAAM+ve+X+7n+JaIpqqQEQkpnrLEI2ISK+jBl5EJKbUwIuIxFSnLrI6544ieNqrmuBK9bUlqZWIiHRa0RdZnXOjCa5Ib0mwssozwJFm9kY7+/UhWLWkAf8eVCmvJMHDHM9ZsKhDpyiuFaVksVVcK0qH49qZHvzuwGNmtgjAOXcXwRJZF7az39ZojcdKsiNZK/Z0guJaeUoRW8W18hQc18408KMI708NNRA8MdaeBoC585bS1KxbNLtLKplgzOh+4MewMxTXClHi2CquFaKYuHamga8iWJOwVYJgvob2NAM0NadpatIXpgKU6me34lp5ShFbxbXyFBzXztxFM5dgPKjVCNqYE0JERMqrMz34fwMXOOeGEaw+fghwQklqJSIinVZ0D97M5gHnEszQ9jLwVzN7tlQVExGRzunUffBm9leCpeRERKTC6ElWEZGYUgMvIhJTauBFRGKqtyz4UXZ/G7JzlB6aXuWVfWmTBVE6nfPkwEGWWU3s2QVvd0ndRKR3UA9eRCSm1MCLiMSUhmiKtO+ILbz8rVv7y0L2v/qnmUxVkkI9NvPxKL3bpL97ZTMWWAdqKCK9nXrwIiIxpQZeRCSm1MCLiMSUxuDbUJOs9vL1+68Tpet+cbpXVjV4dEnOmdx0lyh9/7ZTvLJh95fkFL1CMue6xy5rbRyl92GoV3b42HlRevBVp/rHGbeJf+BEpk/0yVHf9opee3V4lD6h+V2vbPanHxZQa1mdwXUDovSE/iO9snWqB0XpG4/y+6t9Tv91JpPwyxqf8q9vLfzlo1F69txBXtl3mt/LlPWwOKoHLyISU2rgRURiSkM0ObJvf7x5s8+9sn5X/yGTyfnJ94VHUkvg9ufH5rwyq+Tn6Mn2GLG5l9+fIVF6UM7ycoc99/OizpFuyY1rJr/mbTd5JTtkpWc+e59X9qNTMzNp3zCvFEvgxssWQ9eL0ncM6uuVjT5rYpRO7X1cwcf0Y+fHMbXdwV5+xAOZ/Iic48y06VF6+uEPeGV7LHq64Pp0B/XgRURiSg28iEhMqYEXEYkpjcHnuGViZtx9wI03lP38zS//K0r/bumrZT9/pfvzsMxtpIc88l2vrGrwqLz7pZcsitLNM5/wypZc92CUfv7V/MfINTR/9e+YAAAOqklEQVS5wstv/viPonRymwO8sktPy8wMestPa7yylU3+bKO90ZO//VqUTu1xbEmO2bKoPko33Xm9V1b/t0+9/Nr3/SxKJwb4t9Em3XZRerspOddk9tAYvIiIdAM18CIiMaUhmgrz+KTMLVlzP1/YjTWpDEP7DvTyB1+deSI1d0im+b0XovTz37jbK/tt9fIo/eCHL7VxxneKqGXgiZ2uiNJbP3WmV1Z9ZGb45qZLl3plxy54nN4uMby4J8Gzh95m7vQbr2y3xTOj9JJVy2nLAbtlnnr967nreWWpA78fpatGru+V7TUicwvnIx++XECNy0s9eBGRmFIDLyISU2rgRURiSmPwOc56da0offUz93hlqe0Pzt280xr/cpmX/2GLpiPItnCZfztby8zMuGrz8HFe2fj9Ls67Xzns9HHm+smnf77CK6s56cIo/bXRDf6OC+j1Fl/0tyg95O+75d1u5S/9WVy/9c9MH/X++S8Wff77GjLXb1b9179Gkjowk04MXMsr26pqzSj9SNFn7zrqwYuIxFRBPXjn3BrAM8D+ZjbHObc7cDlQB0wxs/O6sI4iIlKEdht459y2wE3ABmG+DpgM7AR8ADzgnNvHzB7qyoqWyy31z0Tp+457zSub+1ZphmjSSxdH6eOvXuSVvbu4IXdzyae5yct2x7BMPsk99stbtvzzmrxlvdXYGZmhyfU3+5ZXlkpkFm/ZpnaMV3b6ykSUbhg6wSt7bfH7Ubq9p4UTicxxEjXJvNs1v/Ocl79y4Yw2j9vdChmiOR44GWh97ncbYJaZzTazJuB24LAuqp+IiBSp3R68mX0XwDnX+tIoILub2QCMQUREKkoxF1mrgOzVFBLkzqYvIiLdrpjbJOcC2SvfjiAzfBMrfVN9SnKcpocme/mLfjE3Sk+pr+wxvEpz3ORlUfrkP+RfhTx7TBVgWM6UB9l2GRj9OuXBj2d6ZZ+vXJa7eV5njNopSifXmZh3u1OXJvKW9VbpdKbP+PYn87yy80btHKXPnXF+3mPkrpPV9HDm/93SO6Z7Za+87K/bNHGrzGLadZdck7+iOWP57U2B0N2KaeBnAM45NwGYDRxFcNFVREQqSIeHaMxsBTAJmAq8AbwF3FXaaomISGcV3IM3s/FZ6WnA5vm37rl+MSqzoMQZPx1WkmPedZ4/gvXbBU/k2VLaM7Uhc5va1Da2+2Bbf9a/IX/PLN7SPMsfFkuuv22m7IUHvbKB37jcyze3NEfpsWv4348L/5J1a2TO8F7Lh+9G6f+t1Cyh5ZC9QPfAnMW6v5a7cYFa/vtYJ2pUfnqSVUQkptTAi4jElBp4EZGY6vWzSWYv4gxw2IyfF7RfIul/dOmcx+Y33fToKP3O4ljeRVpxNh0yPkoPuspfUWnxMcdH6e1fWuKV/bUmMyvlxBf9VYE+2P5eL7/nG5nb5K5J+KsQJSdsHaWzVxoCOOnAP0Xp3NsApW3/aZofpc+Z+4ZXVjVmo7LWpfkDf+rPPqnMtBOVuHi6evAiIjGlBl5EJKZ65RDNlCE7R+kDpv/IL0wXNutC491Xe3l3zjQv/+GST4qqmxRv59rMUEvu4sj/nZmZLmn2p//xyi4ZmRlq+Ru+QXfc6OWfozDzjzzXy99a/1aBe0qup+ZnhmUOOtiPxz13fzdKJ8du0uV1qb3wKi8/d+EJUXrdf/lDbx15CrqrqAcvIhJTauBFRGJKDbyISEz1ijH400f7Dybv++sNonSipq6oY15+iX+7VP3nHxd1HCmdOz55JUr/KmflnX3+sFmUvu4Hfr/mupWZaQSaP/BX8Sp2XPfC+YOL2k++aKPBmWsrfxjUlHe7Vb8/x8uvfKvw/5OrFmVm+Hz/3UFe2eYPfCdK517b6X9d5prAadv+wiv7Zf1/Cj5/V1EPXkQkptTAi4jElBp4EZGYis0Y/DlZq74A/PSyzCPMyYn+dASJ/oWNj7Z88qGX33X3C6P0jAXWwRpKV1u47NMofeLRd3tlf5zxqyj97ecP9sq+3bQyk8mZ5je9/HMv3/xcZjrhqnU39cqyH5u/aoq/Dv3kHZ9pq+rShunHZZ5TqDntEq8se3rnr9zi34f+5qIPSnL+gw7MTBH9lwdO8cqqhq0dpX9ynr9K1K9OSWbqmTXNdDmpBy8iElNq4EVEYio2QzQHNfmPBad2OKSo46SXLo7S9+72R69sxsLCh2XWWzOzLvkufdf1ynZozMxAt/em/s/IEY+9U/A5eqPsW+aO7TPBK9tyZWaoZaPx88mnafo9Xj5RkxmWaX7KX7r52L/4MwT+o+GFKP3quC28svWfzlr9KRmb/1pl96XBY7188qBv5t32zydm4lGqIZlc9zY8H6Uf2f1mr2yfVy6I0qkDTvTK3HmZYbk3Fr3fJXVrj3rwIiIxpQZeRCSm1MCLiMRUjx4ozL41ctNpPyj58fuk017+wJFb5t32mFUDvPxeV2SmQ0jtdETe/VoW/M/LH3HADVF6Sv2MguoZZ5sPWcfL/+egzOdce8HPvLLsVZTe3PVXXtnxW58RpZ9clH/q3uU5q/K0dXvb+OM0HUFX+GDJQi+ffinrusjam3llRx+7IkqfdGmXVguAWgqbThxg17rMLZRvoDF4EREpITXwIiIx1aOHaB5alfnZc95i/6nTRL81izpm9n77veQPAeznbZjzt7HAlaByZT8JBzCe4ma3jKspg/t7+doLrozSTffd4JUdedHbUfr+hhe7pD7XDN81Slcf86O82y39+ZV5y6RtS1Yt9/LpBflnhaz5wUVR+vi//Nwru6n+6ZLUZ2Btvyi9w4/7t7Gl78YP/68k5+8M9eBFRGKqoB68c+584PAw+4CZneWc2x24HKgDppjZeV1URxERKUK7PfiwId8T2AKYCGzpnDsSmAwcCHwJ2No5t09XVlRERDqmkB58A/AjM1sF4Jx7E9gAmGVms8PXbgcOAx7qqoquzssL34vSLe/N9MqqRm9YzqoUrenBP3n5f654N8+WvdPYP38vb9nu57/k5btihs9zc2YpnXTT9plMzmpg2VMgbPucPwulFO+paxqj9C7H5xRmXQv7/a1+H3O9YzNTUJzd8FjR539oQGZVr+pvnZV3u2ab7uXTpPNsWT7tNvBm9npr2jm3PsFQzdUEDX+rBmBMyWsnIiJFK/giq3NuY+BR4MfAe+D9eUpAB54AEBGRLlfoRdavAlOBH5rZnc65nYCRWZuMAOq7oH4Fu/bMt738D+7PPCFataY/ET/V/qIOXaFlUdbiA835Fwr+3oXvefmumhGvp1r1R/9WyOzbJKeu6/8EHuOvg16Ug0Zu5eXPe/psf4OsBUGan73PK7rqtMww4ZxPP+p8ZQSAK2syw1073nWVV5Y69NQondzwq17ZiXtPidJn+5NAtn2+Ebt6+YmPn1zQficc+08v39jG//tyabeBd86NBe4FjjCz1oGsGUGRmwDMBo4iuOgqIiIVopAe/JlALXC5c671teuBSQS9+lrgQeCuLqifiIgUqZCLrKcBp+Up3ry01RERkVLp0VMVZMu9DersLTP5q4b7Y2pfasys/DN2yGde2drTrijq/I13XO7lx53/RJRevGJJUccUuOWBoV7+ez/L3DI35E5/PHbJS49G6Zkn/tcr+7i5Nu85dvhmJj41p+U8r5e7CHfWwt5nn/qcV3ZNg39OKY1HPnw5Sh9/qR+Pmw/Nv1/thZnvx5Kj/GkL5p18p5fvOzDzvRo69Vz/QNnTkmQv0A48s2VmqoQ7Pqm82V81VYGISEypgRcRianYDNG05dSP2niKbVFOftxuXVoX6ZjTP/RjN2jrzK2Rh5w/zCtLHfj9KD3xxf1Lcv6Vl/pPLk68NTODqW6FLL+pH73g5XfYKjNE8u37j/LKqkasF6Vzb6EcN83PF2rZmf7lyN0Wlf7p6VJSD15EJKbUwIuIxJQaeBGRmOoVY/ASH5MWPB6lTzit2ivb4aI3o/TVff2+y9jjhuc95uJ7M9NaXNjg35b5p4bnvXw63f0zBPZmuY//n5J1fe3FA1d4ZVeeuVaUTh2W71GeL1r1+3O8/D9uy8waOmnhrIKPUwnUgxcRiSk18CIiMaUhGumxVjU3evnHPsrM5rhx7sY/y30hn8q+7U3ym1z/jJ8/Iytzxr3lrUyFUA9eRCSm1MCLiMSUGngRkZhSAy8iElNq4EVEYkoNvIhITKmBFxGJKTXwIiIxpQZeRCSmuuNJ1iRAKpnohlNLq6zPP1miQyquFaLEsVVcK0Qxce2OBn4kwJjR/brh1LIaI4F3S3QcxbWylCK2imvlKTiu3dHAPwfsCDQAzd1wfgkkCb4oz5XoeIpr5ShlbBXXytHhuCY0v7WISDzpIquISEypgRcRiSk18CIiMaUGXkQkptTAi4jElBp4EZGYUgMvIhJT3bLotnPuKOA8oBq4wsyu7Y56VALn3PnA4WH2ATM7yzm3O3A5UAdMMbPzuq2CHaC4Ziiu8dWTYlv2HrxzbjRwMbADMBE4wTm3UbnrUQnCL8WewBYEn8WWzrkjgcnAgcCXgK2dc/t0Xy0Lo7hmKK7x1dNi2x1DNLsDj5nZIjNbCtwFHNoN9agEDcCPzGyVmTUCbwIbALPMbLaZNQG3A4d1ZyULpLhmKK7x1aNi2x1DNKMIPqRWDcA23VCPbmdmr7emnXPrE/zsu5ovfj5jyly1YiiuIcU1vnpabLujB18FZE+AkwBauqEeFcM5tzHwKPBj4D165uejuOZQXOOrp8S2Oxr4uYRTkIZGAPXdUI+K4Jz7KjANONvMbqXnfj49td5dQnGNr54U2+4Yovk3cIFzbhiwFDgEOKEb6tHtnHNjgXuBI8zssfDlGUGRmwDMBo4iuIBT6RTXkOIaXz0ttmVv4M1snnPuXOBxoAb4o5k9W+56VIgzgVrgcudc62vXA5OAqWHZgwQXtiqa4upRXOOrR8VW88GLiMSUnmQVEYkpNfAiIjGlBl5EJKbUwIuIxJQaeBGRmFIDLyISU2rgRURiSg28iEhM/X/Ke9IBcw+k7wAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 6 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Probabilities of the wrong predicted numbers\n",
    "Y_pred_errors_prob = np.max(Y_pred_errors,axis = 1)\n",
    "\n",
    "# Predicted probabilities of the true values in the error set\n",
    "true_prob_errors = np.diagonal(np.take(Y_pred_errors, Y_true_errors, axis=1))\n",
    "\n",
    "# Difference between the probability of the predicted label and the true label\n",
    "delta_pred_true_errors = Y_pred_errors_prob - true_prob_errors\n",
    "\n",
    "# Sorted list of the delta prob errors\n",
    "sorted_dela_errors = np.argsort(delta_pred_true_errors)\n",
    "\n",
    "# Top 6 errors \n",
    "most_important_errors = sorted_dela_errors[-6:]\n",
    "\n",
    "# Show the top 6 errors\n",
    "display_errors(most_important_errors, X_val_errors, Y_pred_classes_errors, Y_true_errors)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 预测"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-02-15T14:00:46.911200Z",
     "start_time": "2019-02-15T14:00:13.827112Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "results = model.predict(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-02-15T14:15:59.260082Z",
     "start_time": "2019-02-15T14:15:59.249503Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1.25328206e-14, 1.69472867e-12, 1.00000000e+00, ...,\n",
       "        3.72805520e-09, 4.36439600e-12, 9.22717289e-15],\n",
       "       [9.99997139e-01, 7.67150510e-11, 1.61521818e-08, ...,\n",
       "        6.98757763e-09, 4.94736305e-08, 2.43753948e-06],\n",
       "       [7.38518136e-11, 1.06666564e-10, 2.10796571e-08, ...,\n",
       "        2.43290003e-07, 7.54412986e-06, 9.99988437e-01],\n",
       "       ...,\n",
       "       [2.70637904e-18, 1.89910758e-15, 6.78534702e-13, ...,\n",
       "        1.49611416e-13, 2.83102031e-09, 3.08127052e-10],\n",
       "       [1.25553179e-09, 3.33792105e-09, 9.47844914e-09, ...,\n",
       "        9.96628114e-06, 1.59950829e-07, 9.99963760e-01],\n",
       "       [8.59142384e-15, 8.94977180e-14, 1.00000000e+00, ...,\n",
       "        2.17648677e-11, 2.39016602e-11, 9.64295130e-16]], dtype=float32)"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels = np.argmax(results, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([2, 0, 9, ..., 3, 9, 2])"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "submission = pd.DataFrame({'ImageId':range(1,28001), 'Label':labels}, index=range(1,28001))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "submission.to_csv('cnn_mnist_datagen.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 总结"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 为什么使用这样的网络结构"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1、数据集图像决定复杂度：\n",
    "\n",
    "> 图像由小尺寸的数字图像组成 + 图像比较标准化 => 问题简单、不需要非常深的网络结构\n",
    "\n",
    "> Conv + relu + maxpool => 降低过拟合，可以指数级增加过滤器的数量\n",
    "\n",
    "2、更多尝试：\n",
    "\n",
    "> 根据实际任务选择合适的网络架构：最新论文/ImageNet => LeNet、AlexNet\n",
    "\n",
    "> There are many Convolution neural networks models proposed in many papers . Every model gives better accuracy than the one before it . that is Alex net performs better than lenet and googLeNet is better than AlexNet but in general with some error analysis and trials you should find the number of layers and the architecture that will fit the task\n",
    "\n",
    "> You may try to find a paper or an algorithm that was proven to work well for similar tasks. you then try to modify it to fit your task according to the results you get from the algorithm. You may also consider not to reinvent the wheel by implementing the algorithm from scratch, Instead you may use one of well known algorithm used in ImageNet or other challenges like VGG-16 , VGG-19 or yolo depending on the Task. Transfer learning makes it easier for the training process as the algorithm will be pre-trained but you will have to decide how many layers you want to freeze according to the training data you have.\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 【亮点🦍】：数据增强 => 减少过拟合"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##  解释性"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1、混淆矩阵"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "seaborn绘制混淆矩阵： https://seaborn.pydata.org/generated/seaborn.heatmap.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "py27",
   "language": "python",
   "name": "py27"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.15"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "165px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
